{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"MNIST_data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(\"float\", [None, 784])\n",
    "y = tf.placeholder(\"int64\", [None])\n",
    "learning_rate = tf.placeholder(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 高斯分布初始化\n",
    "def initialize(shape, stddev=0.1):\n",
    "    return tf.truncated_normal(shape, stddev=0.1)\n",
    "# Xavier初始化\n",
    "def Xavier(node_in, node_out):\n",
    "    return np.random.randn(node_in, node_out).astype('float32') / np.sqrt(node_in)\n",
    "\n",
    "# MSRA初始化(He 初始化)\n",
    "def MSRA(node_in, node_out):\n",
    "    return np.random.randn(node_in,node_out).astype('float32') / np.sqrt(node_in/2)\n",
    "\n",
    "# 构造神经网络，hidden1为与输入层连接的隐藏层，**kwags为可变参数，可以传入任意数目的隐层，initial_way为参数初始化方式\n",
    "def create_network(hidden1, initial_way, **kwags):\n",
    "    hiddens = [hidden1]\n",
    "    if initial_way == initialize:\n",
    "        W = tf.Variable(initial_way([784, hidden1]))\n",
    "        b = tf.Variable(initial_way([hidden1]))\n",
    "    else:\n",
    "        W = tf.Variable(initial_way(784, hidden1))\n",
    "        b = tf.Variable(initial_way(1, hidden1))\n",
    "    logits = tf.matmul(x, W) + b\n",
    "    output = tf.nn.relu(logits)\n",
    "    if kwags != '':\n",
    "        for k in kwags:\n",
    "            if initial_way == initialize:\n",
    "                W = tf.Variable(initial_way([hiddens[-1], kwags[k]]))\n",
    "                b = tf.Variable(initial_way([kwags[k]]))\n",
    "            else:\n",
    "                W = tf.Variable(initial_way(hiddens[-1], kwags[k]))\n",
    "                b = tf.Variable(initial_way(1, kwags[k]))\n",
    "            logits = tf.matmul(output, W) + b\n",
    "            output = tf.nn.relu(logits)\n",
    "            hiddens.append(kwags[k])\n",
    "    if initial_way == initialize:\n",
    "        W = tf.Variable(initial_way([hiddens[-1], 10]))\n",
    "        b = tf.Variable(initial_way([10]))\n",
    "    else:\n",
    "        W = tf.Variable(initial_way(hiddens[-1], 10))\n",
    "        b = tf.Variable(initial_way(1, 10))\n",
    "    logits = tf.matmul(output, W) + b\n",
    "    \n",
    "    return logits,W\n",
    "\n",
    "# hidden1为与输入层连接的隐藏层，**kwags为可变参数，可以传入任意数目的隐层，\n",
    "# lambda_flag为正则参数，l1/l2，默认为None，lambda_value为正则系数\n",
    "# initial_way为参数初始化方式(默认为高斯分布)，上面定义了3种\n",
    "def train(batch_size, training_step, lr, hidden1, lambda_flag=None, lambda_value=0.1, \n",
    "          initial_way=initialize, **kwags):\n",
    "    \n",
    "    logits,W = create_network(hidden1, initial_way, **kwags)\n",
    "    if lambda_flag == 'l1':\n",
    "        cross_entropy_loss = tf.reduce_mean(\n",
    "            tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "            + tf.contrib.layers.l1_regularizer(lambda_value)(W))\n",
    "    if lambda_flag == 'l2':\n",
    "        cross_entropy_loss = tf.reduce_mean(\n",
    "            tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "            + tf.contrib.layers.l2_regularizer(lambda_value)(W))\n",
    "    if lambda_flag is None:\n",
    "        cross_entropy_loss = tf.reduce_mean(\n",
    "            tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "        \n",
    "    optimizer = tf.train.GradientDescentOptimizer(\n",
    "        learning_rate = learning_rate).minimize(cross_entropy_loss)\n",
    "    pred = tf.nn.softmax(logits)\n",
    "    correct_pred = tf.equal(tf.argmax(pred, 1), y)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        #定义训练集、验证集与测试集\n",
    "        train_data = {\n",
    "            x: mnist.train.images,\n",
    "            y: mnist.train.labels,\n",
    "        }\n",
    "        validate_data = {\n",
    "            x: mnist.validation.images,\n",
    "            y: mnist.validation.labels,\n",
    "        }\n",
    "        test_data = {x: mnist.test.images, y: mnist.test.labels}\n",
    "\n",
    "        for i in range(training_step):\n",
    "            xs, ys = mnist.train.next_batch(batch_size)\n",
    "            _, loss = sess.run(\n",
    "                [optimizer, cross_entropy_loss],\n",
    "                feed_dict={\n",
    "                    x: xs,\n",
    "                    y: ys,\n",
    "                    learning_rate: lr\n",
    "                })\n",
    "\n",
    "            #每100次训练打印一次损失值与验证准确率\n",
    "            if (i+1) > 0 and (i+1) % 100 == 0:\n",
    "                validate_accuracy = sess.run(accuracy, feed_dict=validate_data)\n",
    "                print(\n",
    "                    \"after %d training steps, the loss is %g, the validation accuracy is %g\"\n",
    "                    % (i+1, loss, validate_accuracy))\n",
    "\n",
    "        print(\"the training is finish!\")\n",
    "        \n",
    "        #最终的训练和测试准确率\n",
    "        acc_train = sess.run(accuracy, feed_dict=train_data)\n",
    "        acc_test = sess.run(accuracy, feed_dict=test_data)\n",
    "        print(\"the train accuarcy is:\", acc_train)\n",
    "        print(\"the test accuarcy is:\", acc_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 初始状态与示例代码一致"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.211828, the validation accuracy is 0.857\n",
      "after 200 training steps, the loss is 0.139349, the validation accuracy is 0.915\n",
      "after 300 training steps, the loss is 0.624946, the validation accuracy is 0.8848\n",
      "after 400 training steps, the loss is 0.299957, the validation accuracy is 0.9276\n",
      "after 500 training steps, the loss is 0.216958, the validation accuracy is 0.9382\n",
      "after 600 training steps, the loss is 0.546945, the validation accuracy is 0.9388\n",
      "after 700 training steps, the loss is 0.301399, the validation accuracy is 0.9464\n",
      "after 800 training steps, the loss is 0.112065, the validation accuracy is 0.9518\n",
      "after 900 training steps, the loss is 0.447516, the validation accuracy is 0.956\n",
      "after 1000 training steps, the loss is 0.137875, the validation accuracy is 0.957\n",
      "the training is finish!\n",
      "the test accuarcy is: 0.9503\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=1000, lr=0.3, hidden1=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 增加训练轮次并降低学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.448116, the validation accuracy is 0.8468\n",
      "after 200 training steps, the loss is 0.377825, the validation accuracy is 0.8786\n",
      "after 300 training steps, the loss is 0.47967, the validation accuracy is 0.8862\n",
      "after 400 training steps, the loss is 0.179951, the validation accuracy is 0.905\n",
      "after 500 training steps, the loss is 0.347464, the validation accuracy is 0.9112\n",
      "after 600 training steps, the loss is 0.206758, the validation accuracy is 0.9164\n",
      "after 700 training steps, the loss is 0.467023, the validation accuracy is 0.9186\n",
      "after 800 training steps, the loss is 0.312322, the validation accuracy is 0.9208\n",
      "after 900 training steps, the loss is 0.606184, the validation accuracy is 0.9234\n",
      "after 1000 training steps, the loss is 0.401586, the validation accuracy is 0.9338\n",
      "after 1100 training steps, the loss is 0.446529, the validation accuracy is 0.9344\n",
      "after 1200 training steps, the loss is 0.0994114, the validation accuracy is 0.9388\n",
      "after 1300 training steps, the loss is 0.19659, the validation accuracy is 0.9364\n",
      "after 1400 training steps, the loss is 0.144689, the validation accuracy is 0.9434\n",
      "after 1500 training steps, the loss is 0.246027, the validation accuracy is 0.9348\n",
      "after 1600 training steps, the loss is 0.248595, the validation accuracy is 0.9454\n",
      "after 1700 training steps, the loss is 0.205904, the validation accuracy is 0.9426\n",
      "after 1800 training steps, the loss is 0.271872, the validation accuracy is 0.9476\n",
      "after 1900 training steps, the loss is 0.190341, the validation accuracy is 0.954\n",
      "after 2000 training steps, the loss is 0.175279, the validation accuracy is 0.9468\n",
      "after 2100 training steps, the loss is 0.414509, the validation accuracy is 0.9528\n",
      "after 2200 training steps, the loss is 0.070283, the validation accuracy is 0.95\n",
      "after 2300 training steps, the loss is 0.29142, the validation accuracy is 0.9558\n",
      "after 2400 training steps, the loss is 0.026355, the validation accuracy is 0.956\n",
      "after 2500 training steps, the loss is 0.140508, the validation accuracy is 0.9562\n",
      "after 2600 training steps, the loss is 0.195527, the validation accuracy is 0.9528\n",
      "after 2700 training steps, the loss is 0.0746088, the validation accuracy is 0.9576\n",
      "after 2800 training steps, the loss is 0.135918, the validation accuracy is 0.9586\n",
      "after 2900 training steps, the loss is 0.178772, the validation accuracy is 0.956\n",
      "after 3000 training steps, the loss is 0.290466, the validation accuracy is 0.9584\n",
      "after 3100 training steps, the loss is 0.123245, the validation accuracy is 0.9572\n",
      "after 3200 training steps, the loss is 0.0193857, the validation accuracy is 0.9624\n",
      "after 3300 training steps, the loss is 0.0706061, the validation accuracy is 0.9596\n",
      "after 3400 training steps, the loss is 0.139942, the validation accuracy is 0.9644\n",
      "after 3500 training steps, the loss is 0.0772649, the validation accuracy is 0.9628\n",
      "after 3600 training steps, the loss is 0.122348, the validation accuracy is 0.9614\n",
      "after 3700 training steps, the loss is 0.101513, the validation accuracy is 0.9632\n",
      "after 3800 training steps, the loss is 0.116096, the validation accuracy is 0.9632\n",
      "after 3900 training steps, the loss is 0.093191, the validation accuracy is 0.964\n",
      "after 4000 training steps, the loss is 0.0687291, the validation accuracy is 0.9614\n",
      "after 4100 training steps, the loss is 0.027076, the validation accuracy is 0.9658\n",
      "after 4200 training steps, the loss is 0.0275718, the validation accuracy is 0.968\n",
      "after 4300 training steps, the loss is 0.0362397, the validation accuracy is 0.9676\n",
      "after 4400 training steps, the loss is 0.0973799, the validation accuracy is 0.9632\n",
      "after 4500 training steps, the loss is 0.0608562, the validation accuracy is 0.9694\n",
      "after 4600 training steps, the loss is 0.0805785, the validation accuracy is 0.968\n",
      "after 4700 training steps, the loss is 0.0697457, the validation accuracy is 0.966\n",
      "after 4800 training steps, the loss is 0.113572, the validation accuracy is 0.9668\n",
      "after 4900 training steps, the loss is 0.227151, the validation accuracy is 0.9696\n",
      "after 5000 training steps, the loss is 0.1196, the validation accuracy is 0.9688\n",
      "after 5100 training steps, the loss is 0.0657466, the validation accuracy is 0.9692\n",
      "after 5200 training steps, the loss is 0.0198177, the validation accuracy is 0.9702\n",
      "after 5300 training steps, the loss is 0.0396907, the validation accuracy is 0.9728\n",
      "after 5400 training steps, the loss is 0.125021, the validation accuracy is 0.966\n",
      "after 5500 training steps, the loss is 0.027286, the validation accuracy is 0.9654\n",
      "after 5600 training steps, the loss is 0.0857107, the validation accuracy is 0.967\n",
      "after 5700 training steps, the loss is 0.247792, the validation accuracy is 0.965\n",
      "after 5800 training steps, the loss is 0.143048, the validation accuracy is 0.9688\n",
      "after 5900 training steps, the loss is 0.0627786, the validation accuracy is 0.9692\n",
      "after 6000 training steps, the loss is 0.0424572, the validation accuracy is 0.9694\n",
      "after 6100 training steps, the loss is 0.244158, the validation accuracy is 0.9712\n",
      "after 6200 training steps, the loss is 0.176122, the validation accuracy is 0.9716\n",
      "after 6300 training steps, the loss is 0.305692, the validation accuracy is 0.9694\n",
      "after 6400 training steps, the loss is 0.163361, the validation accuracy is 0.972\n",
      "after 6500 training steps, the loss is 0.0153387, the validation accuracy is 0.973\n",
      "after 6600 training steps, the loss is 0.181962, the validation accuracy is 0.9714\n",
      "after 6700 training steps, the loss is 0.139517, the validation accuracy is 0.9726\n",
      "after 6800 training steps, the loss is 0.070602, the validation accuracy is 0.97\n",
      "after 6900 training steps, the loss is 0.069369, the validation accuracy is 0.9692\n",
      "after 7000 training steps, the loss is 0.114622, the validation accuracy is 0.9716\n",
      "after 7100 training steps, the loss is 0.208657, the validation accuracy is 0.9678\n",
      "after 7200 training steps, the loss is 0.0766836, the validation accuracy is 0.973\n",
      "after 7300 training steps, the loss is 0.0534874, the validation accuracy is 0.9736\n",
      "after 7400 training steps, the loss is 0.312768, the validation accuracy is 0.9728\n",
      "after 7500 training steps, the loss is 0.0283231, the validation accuracy is 0.975\n",
      "after 7600 training steps, the loss is 0.128297, the validation accuracy is 0.9758\n",
      "after 7700 training steps, the loss is 0.0966312, the validation accuracy is 0.9742\n",
      "after 7800 training steps, the loss is 0.0250484, the validation accuracy is 0.9724\n",
      "after 7900 training steps, the loss is 0.240957, the validation accuracy is 0.9704\n",
      "after 8000 training steps, the loss is 0.0321455, the validation accuracy is 0.9752\n",
      "after 8100 training steps, the loss is 0.0789187, the validation accuracy is 0.974\n",
      "after 8200 training steps, the loss is 0.0766057, the validation accuracy is 0.973\n",
      "after 8300 training steps, the loss is 0.0300064, the validation accuracy is 0.9706\n",
      "after 8400 training steps, the loss is 0.0537146, the validation accuracy is 0.9718\n",
      "after 8500 training steps, the loss is 0.0497828, the validation accuracy is 0.9706\n",
      "after 8600 training steps, the loss is 0.0356644, the validation accuracy is 0.9744\n",
      "after 8700 training steps, the loss is 0.0725217, the validation accuracy is 0.9738\n",
      "after 8800 training steps, the loss is 0.0108273, the validation accuracy is 0.9742\n",
      "after 8900 training steps, the loss is 0.0717377, the validation accuracy is 0.9754\n",
      "after 9000 training steps, the loss is 0.00883691, the validation accuracy is 0.9742\n",
      "after 9100 training steps, the loss is 0.0638007, the validation accuracy is 0.9716\n",
      "after 9200 training steps, the loss is 0.00639676, the validation accuracy is 0.9754\n",
      "after 9300 training steps, the loss is 0.0153521, the validation accuracy is 0.9722\n",
      "after 9400 training steps, the loss is 0.0352664, the validation accuracy is 0.9756\n",
      "after 9500 training steps, the loss is 0.111126, the validation accuracy is 0.9764\n",
      "after 9600 training steps, the loss is 0.0499683, the validation accuracy is 0.974\n",
      "after 9700 training steps, the loss is 0.025052, the validation accuracy is 0.9762\n",
      "after 9800 training steps, the loss is 0.0388996, the validation accuracy is 0.9754\n",
      "after 9900 training steps, the loss is 0.284807, the validation accuracy is 0.9728\n",
      "after 10000 training steps, the loss is 0.0625912, the validation accuracy is 0.973\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10100 training steps, the loss is 0.138324, the validation accuracy is 0.9762\n",
      "after 10200 training steps, the loss is 0.193072, the validation accuracy is 0.9738\n",
      "after 10300 training steps, the loss is 0.0099605, the validation accuracy is 0.974\n",
      "after 10400 training steps, the loss is 0.0280843, the validation accuracy is 0.9742\n",
      "after 10500 training steps, the loss is 0.0507463, the validation accuracy is 0.9754\n",
      "after 10600 training steps, the loss is 0.0290385, the validation accuracy is 0.9772\n",
      "after 10700 training steps, the loss is 0.054515, the validation accuracy is 0.9764\n",
      "after 10800 training steps, the loss is 0.0226385, the validation accuracy is 0.975\n",
      "after 10900 training steps, the loss is 0.219368, the validation accuracy is 0.9732\n",
      "after 11000 training steps, the loss is 0.0916229, the validation accuracy is 0.9756\n",
      "after 11100 training steps, the loss is 0.132807, the validation accuracy is 0.9772\n",
      "after 11200 training steps, the loss is 0.0927798, the validation accuracy is 0.9764\n",
      "after 11300 training steps, the loss is 0.160379, the validation accuracy is 0.9768\n",
      "after 11400 training steps, the loss is 0.028586, the validation accuracy is 0.9788\n",
      "after 11500 training steps, the loss is 0.0408853, the validation accuracy is 0.9764\n",
      "after 11600 training steps, the loss is 0.0463966, the validation accuracy is 0.9748\n",
      "after 11700 training steps, the loss is 0.0138167, the validation accuracy is 0.9756\n",
      "after 11800 training steps, the loss is 0.0604791, the validation accuracy is 0.9764\n",
      "after 11900 training steps, the loss is 0.00466668, the validation accuracy is 0.977\n",
      "after 12000 training steps, the loss is 0.0318072, the validation accuracy is 0.9744\n",
      "after 12100 training steps, the loss is 0.0215106, the validation accuracy is 0.9754\n",
      "after 12200 training steps, the loss is 0.0293782, the validation accuracy is 0.9762\n",
      "after 12300 training steps, the loss is 0.027574, the validation accuracy is 0.976\n",
      "after 12400 training steps, the loss is 0.0802172, the validation accuracy is 0.9774\n",
      "after 12500 training steps, the loss is 0.00440539, the validation accuracy is 0.977\n",
      "after 12600 training steps, the loss is 0.0227307, the validation accuracy is 0.9758\n",
      "after 12700 training steps, the loss is 0.0214391, the validation accuracy is 0.9762\n",
      "after 12800 training steps, the loss is 0.179005, the validation accuracy is 0.9772\n",
      "after 12900 training steps, the loss is 0.0502715, the validation accuracy is 0.9778\n",
      "after 13000 training steps, the loss is 0.08875, the validation accuracy is 0.9796\n",
      "after 13100 training steps, the loss is 0.0111331, the validation accuracy is 0.9792\n",
      "after 13200 training steps, the loss is 0.0504506, the validation accuracy is 0.9748\n",
      "after 13300 training steps, the loss is 0.0325848, the validation accuracy is 0.9758\n",
      "after 13400 training steps, the loss is 0.0448845, the validation accuracy is 0.977\n",
      "after 13500 training steps, the loss is 0.0121885, the validation accuracy is 0.976\n",
      "after 13600 training steps, the loss is 0.0446373, the validation accuracy is 0.974\n",
      "after 13700 training steps, the loss is 0.0606898, the validation accuracy is 0.976\n",
      "after 13800 training steps, the loss is 0.0916239, the validation accuracy is 0.977\n",
      "after 13900 training steps, the loss is 0.0109743, the validation accuracy is 0.9784\n",
      "after 14000 training steps, the loss is 0.034913, the validation accuracy is 0.9782\n",
      "after 14100 training steps, the loss is 0.0251328, the validation accuracy is 0.9774\n",
      "after 14200 training steps, the loss is 0.066702, the validation accuracy is 0.9784\n",
      "after 14300 training steps, the loss is 0.0041187, the validation accuracy is 0.9756\n",
      "after 14400 training steps, the loss is 0.0288168, the validation accuracy is 0.9774\n",
      "after 14500 training steps, the loss is 0.0152387, the validation accuracy is 0.9778\n",
      "after 14600 training steps, the loss is 0.0156641, the validation accuracy is 0.978\n",
      "after 14700 training steps, the loss is 0.0289506, the validation accuracy is 0.9784\n",
      "after 14800 training steps, the loss is 0.0953514, the validation accuracy is 0.979\n",
      "after 14900 training steps, the loss is 0.0105428, the validation accuracy is 0.9778\n",
      "after 15000 training steps, the loss is 0.0741741, the validation accuracy is 0.9774\n",
      "after 15100 training steps, the loss is 0.0050025, the validation accuracy is 0.9778\n",
      "after 15200 training steps, the loss is 0.0110622, the validation accuracy is 0.9776\n",
      "after 15300 training steps, the loss is 0.0282019, the validation accuracy is 0.9762\n",
      "after 15400 training steps, the loss is 0.00498936, the validation accuracy is 0.9766\n",
      "after 15500 training steps, the loss is 0.0159718, the validation accuracy is 0.9778\n",
      "after 15600 training steps, the loss is 0.051374, the validation accuracy is 0.9762\n",
      "after 15700 training steps, the loss is 0.0187334, the validation accuracy is 0.9794\n",
      "after 15800 training steps, the loss is 0.0200487, the validation accuracy is 0.9784\n",
      "after 15900 training steps, the loss is 0.0446834, the validation accuracy is 0.9784\n",
      "after 16000 training steps, the loss is 0.00490484, the validation accuracy is 0.9792\n",
      "after 16100 training steps, the loss is 0.00152204, the validation accuracy is 0.979\n",
      "after 16200 training steps, the loss is 0.0183603, the validation accuracy is 0.9774\n",
      "after 16300 training steps, the loss is 0.0493197, the validation accuracy is 0.978\n",
      "after 16400 training steps, the loss is 0.0306997, the validation accuracy is 0.9798\n",
      "after 16500 training steps, the loss is 0.0437495, the validation accuracy is 0.9788\n",
      "after 16600 training steps, the loss is 0.0937082, the validation accuracy is 0.9784\n",
      "after 16700 training steps, the loss is 0.0543162, the validation accuracy is 0.9788\n",
      "after 16800 training steps, the loss is 0.0190897, the validation accuracy is 0.9774\n",
      "after 16900 training steps, the loss is 0.0351258, the validation accuracy is 0.9798\n",
      "after 17000 training steps, the loss is 0.00388418, the validation accuracy is 0.9758\n",
      "after 17100 training steps, the loss is 0.0493537, the validation accuracy is 0.9786\n",
      "after 17200 training steps, the loss is 0.0210216, the validation accuracy is 0.9786\n",
      "after 17300 training steps, the loss is 0.0734584, the validation accuracy is 0.9778\n",
      "after 17400 training steps, the loss is 0.00384265, the validation accuracy is 0.9768\n",
      "after 17500 training steps, the loss is 0.0152044, the validation accuracy is 0.978\n",
      "after 17600 training steps, the loss is 0.0147962, the validation accuracy is 0.9776\n",
      "after 17700 training steps, the loss is 0.0340587, the validation accuracy is 0.9796\n",
      "after 17800 training steps, the loss is 0.0182389, the validation accuracy is 0.9792\n",
      "after 17900 training steps, the loss is 0.0358091, the validation accuracy is 0.9788\n",
      "after 18000 training steps, the loss is 0.00906021, the validation accuracy is 0.9796\n",
      "after 18100 training steps, the loss is 0.00143074, the validation accuracy is 0.9766\n",
      "after 18200 training steps, the loss is 0.013465, the validation accuracy is 0.978\n",
      "after 18300 training steps, the loss is 0.137227, the validation accuracy is 0.9772\n",
      "after 18400 training steps, the loss is 0.023794, the validation accuracy is 0.9762\n",
      "after 18500 training steps, the loss is 0.00965007, the validation accuracy is 0.9784\n",
      "after 18600 training steps, the loss is 0.00826721, the validation accuracy is 0.9788\n",
      "after 18700 training steps, the loss is 0.0581816, the validation accuracy is 0.978\n",
      "after 18800 training steps, the loss is 0.0210186, the validation accuracy is 0.9806\n",
      "after 18900 training steps, the loss is 0.0221928, the validation accuracy is 0.9754\n",
      "after 19000 training steps, the loss is 0.0112269, the validation accuracy is 0.9792\n",
      "after 19100 training steps, the loss is 0.00675719, the validation accuracy is 0.98\n",
      "after 19200 training steps, the loss is 0.0142697, the validation accuracy is 0.98\n",
      "after 19300 training steps, the loss is 0.0423514, the validation accuracy is 0.9782\n",
      "after 19400 training steps, the loss is 0.0392099, the validation accuracy is 0.9782\n",
      "after 19500 training steps, the loss is 0.0236014, the validation accuracy is 0.9762\n",
      "after 19600 training steps, the loss is 0.133383, the validation accuracy is 0.978\n",
      "after 19700 training steps, the loss is 0.0359386, the validation accuracy is 0.9784\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19800 training steps, the loss is 0.0155682, the validation accuracy is 0.9778\n",
      "after 19900 training steps, the loss is 0.0188753, the validation accuracy is 0.9798\n",
      "after 20000 training steps, the loss is 0.04678, the validation accuracy is 0.9792\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.9945273\n",
      "the test accuarcy is: 0.9776\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=20000, lr=0.1, hidden1=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "增加训练轮次后，模型效果达到了0.97以上"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 增加隐藏层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.388337, the validation accuracy is 0.784\n",
      "after 200 training steps, the loss is 0.572253, the validation accuracy is 0.8662\n",
      "after 300 training steps, the loss is 0.344605, the validation accuracy is 0.9096\n",
      "after 400 training steps, the loss is 0.391782, the validation accuracy is 0.9124\n",
      "after 500 training steps, the loss is 0.208882, the validation accuracy is 0.9324\n",
      "after 600 training steps, the loss is 0.227753, the validation accuracy is 0.9328\n",
      "after 700 training steps, the loss is 0.100671, the validation accuracy is 0.9294\n",
      "after 800 training steps, the loss is 0.515145, the validation accuracy is 0.9354\n",
      "after 900 training steps, the loss is 0.458564, the validation accuracy is 0.9432\n",
      "after 1000 training steps, the loss is 0.315044, the validation accuracy is 0.9228\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.9128364\n",
      "the test accuarcy is: 0.9131\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=1000, lr=0.3, hidden1=100, hidden2=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "增加隐藏层后，与初始模型同样的迭代次数和学习率下训练，模型效果反而下降了，应该是模型还未收敛，我们增加迭代次数并且降低学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.559593, the validation accuracy is 0.8072\n",
      "after 200 training steps, the loss is 0.260358, the validation accuracy is 0.873\n",
      "after 300 training steps, the loss is 0.613069, the validation accuracy is 0.9044\n",
      "after 400 training steps, the loss is 0.211086, the validation accuracy is 0.9142\n",
      "after 500 training steps, the loss is 0.1955, the validation accuracy is 0.9132\n",
      "after 600 training steps, the loss is 0.338651, the validation accuracy is 0.9308\n",
      "after 700 training steps, the loss is 0.522671, the validation accuracy is 0.9302\n",
      "after 800 training steps, the loss is 0.244158, the validation accuracy is 0.9272\n",
      "after 900 training steps, the loss is 0.0403984, the validation accuracy is 0.9374\n",
      "after 1000 training steps, the loss is 0.418169, the validation accuracy is 0.943\n",
      "after 1100 training steps, the loss is 0.27381, the validation accuracy is 0.939\n",
      "after 1200 training steps, the loss is 0.464829, the validation accuracy is 0.9434\n",
      "after 1300 training steps, the loss is 0.173809, the validation accuracy is 0.9434\n",
      "after 1400 training steps, the loss is 0.226653, the validation accuracy is 0.9468\n",
      "after 1500 training steps, the loss is 0.121828, the validation accuracy is 0.9494\n",
      "after 1600 training steps, the loss is 0.0926102, the validation accuracy is 0.9478\n",
      "after 1700 training steps, the loss is 0.308926, the validation accuracy is 0.953\n",
      "after 1800 training steps, the loss is 0.475589, the validation accuracy is 0.9498\n",
      "after 1900 training steps, the loss is 0.182247, the validation accuracy is 0.9546\n",
      "after 2000 training steps, the loss is 0.243742, the validation accuracy is 0.9592\n",
      "after 2100 training steps, the loss is 0.107576, the validation accuracy is 0.9598\n",
      "after 2200 training steps, the loss is 0.0202157, the validation accuracy is 0.9628\n",
      "after 2300 training steps, the loss is 0.422681, the validation accuracy is 0.958\n",
      "after 2400 training steps, the loss is 0.113876, the validation accuracy is 0.964\n",
      "after 2500 training steps, the loss is 0.170644, the validation accuracy is 0.9572\n",
      "after 2600 training steps, the loss is 0.0553503, the validation accuracy is 0.956\n",
      "after 2700 training steps, the loss is 0.135349, the validation accuracy is 0.9604\n",
      "after 2800 training steps, the loss is 0.0927045, the validation accuracy is 0.9642\n",
      "after 2900 training steps, the loss is 0.0291913, the validation accuracy is 0.9632\n",
      "after 3000 training steps, the loss is 0.229049, the validation accuracy is 0.9614\n",
      "after 3100 training steps, the loss is 0.00813743, the validation accuracy is 0.9666\n",
      "after 3200 training steps, the loss is 0.316746, the validation accuracy is 0.9592\n",
      "after 3300 training steps, the loss is 0.0502978, the validation accuracy is 0.9632\n",
      "after 3400 training steps, the loss is 0.0845356, the validation accuracy is 0.9658\n",
      "after 3500 training steps, the loss is 0.0200097, the validation accuracy is 0.9654\n",
      "after 3600 training steps, the loss is 0.0671023, the validation accuracy is 0.9602\n",
      "after 3700 training steps, the loss is 0.0744917, the validation accuracy is 0.9686\n",
      "after 3800 training steps, the loss is 0.0101826, the validation accuracy is 0.963\n",
      "after 3900 training steps, the loss is 0.0369638, the validation accuracy is 0.9638\n",
      "after 4000 training steps, the loss is 0.134407, the validation accuracy is 0.966\n",
      "after 4100 training steps, the loss is 0.160023, the validation accuracy is 0.9684\n",
      "after 4200 training steps, the loss is 0.0862213, the validation accuracy is 0.9672\n",
      "after 4300 training steps, the loss is 0.380656, the validation accuracy is 0.9664\n",
      "after 4400 training steps, the loss is 0.209127, the validation accuracy is 0.966\n",
      "after 4500 training steps, the loss is 0.0419595, the validation accuracy is 0.9636\n",
      "after 4600 training steps, the loss is 0.0100538, the validation accuracy is 0.9708\n",
      "after 4700 training steps, the loss is 0.272366, the validation accuracy is 0.966\n",
      "after 4800 training steps, the loss is 0.240821, the validation accuracy is 0.9578\n",
      "after 4900 training steps, the loss is 0.116767, the validation accuracy is 0.971\n",
      "after 5000 training steps, the loss is 0.162924, the validation accuracy is 0.9628\n",
      "after 5100 training steps, the loss is 0.124377, the validation accuracy is 0.9672\n",
      "after 5200 training steps, the loss is 0.0435038, the validation accuracy is 0.9706\n",
      "after 5300 training steps, the loss is 0.165701, the validation accuracy is 0.9694\n",
      "after 5400 training steps, the loss is 0.0586656, the validation accuracy is 0.9704\n",
      "after 5500 training steps, the loss is 0.184343, the validation accuracy is 0.968\n",
      "after 5600 training steps, the loss is 0.0726358, the validation accuracy is 0.968\n",
      "after 5700 training steps, the loss is 0.0243258, the validation accuracy is 0.9726\n",
      "after 5800 training steps, the loss is 0.0533932, the validation accuracy is 0.9694\n",
      "after 5900 training steps, the loss is 0.0479971, the validation accuracy is 0.9702\n",
      "after 6000 training steps, the loss is 0.0196614, the validation accuracy is 0.9744\n",
      "after 6100 training steps, the loss is 0.0019056, the validation accuracy is 0.9698\n",
      "after 6200 training steps, the loss is 0.191683, the validation accuracy is 0.9696\n",
      "after 6300 training steps, the loss is 0.0152664, the validation accuracy is 0.9736\n",
      "after 6400 training steps, the loss is 0.071274, the validation accuracy is 0.9742\n",
      "after 6500 training steps, the loss is 0.0832768, the validation accuracy is 0.9746\n",
      "after 6600 training steps, the loss is 0.127932, the validation accuracy is 0.975\n",
      "after 6700 training steps, the loss is 0.242777, the validation accuracy is 0.9646\n",
      "after 6800 training steps, the loss is 0.0194465, the validation accuracy is 0.9722\n",
      "after 6900 training steps, the loss is 0.0185537, the validation accuracy is 0.9724\n",
      "after 7000 training steps, the loss is 0.145126, the validation accuracy is 0.971\n",
      "after 7100 training steps, the loss is 0.0530021, the validation accuracy is 0.973\n",
      "after 7200 training steps, the loss is 0.122725, the validation accuracy is 0.971\n",
      "after 7300 training steps, the loss is 0.0325384, the validation accuracy is 0.9714\n",
      "after 7400 training steps, the loss is 0.0213306, the validation accuracy is 0.9696\n",
      "after 7500 training steps, the loss is 0.00688903, the validation accuracy is 0.9718\n",
      "after 7600 training steps, the loss is 0.15247, the validation accuracy is 0.9716\n",
      "after 7700 training steps, the loss is 0.00301411, the validation accuracy is 0.9748\n",
      "after 7800 training steps, the loss is 0.0915017, the validation accuracy is 0.9706\n",
      "after 7900 training steps, the loss is 0.186796, the validation accuracy is 0.9702\n",
      "after 8000 training steps, the loss is 0.128321, the validation accuracy is 0.9748\n",
      "after 8100 training steps, the loss is 0.051377, the validation accuracy is 0.9724\n",
      "after 8200 training steps, the loss is 0.105391, the validation accuracy is 0.9722\n",
      "after 8300 training steps, the loss is 0.00361973, the validation accuracy is 0.9744\n",
      "after 8400 training steps, the loss is 0.0241557, the validation accuracy is 0.9738\n",
      "after 8500 training steps, the loss is 0.0347825, the validation accuracy is 0.974\n",
      "after 8600 training steps, the loss is 0.0237504, the validation accuracy is 0.9738\n",
      "after 8700 training steps, the loss is 0.032029, the validation accuracy is 0.975\n",
      "after 8800 training steps, the loss is 0.00589527, the validation accuracy is 0.9738\n",
      "after 8900 training steps, the loss is 0.0126096, the validation accuracy is 0.9734\n",
      "after 9000 training steps, the loss is 0.219967, the validation accuracy is 0.9686\n",
      "after 9100 training steps, the loss is 0.134148, the validation accuracy is 0.9686\n",
      "after 9200 training steps, the loss is 0.0166204, the validation accuracy is 0.9746\n",
      "after 9300 training steps, the loss is 0.0182307, the validation accuracy is 0.975\n",
      "after 9400 training steps, the loss is 0.0588628, the validation accuracy is 0.9776\n",
      "after 9500 training steps, the loss is 0.0549344, the validation accuracy is 0.9746\n",
      "after 9600 training steps, the loss is 0.0235316, the validation accuracy is 0.973\n",
      "after 9700 training steps, the loss is 0.0708328, the validation accuracy is 0.9766\n",
      "after 9800 training steps, the loss is 0.0850729, the validation accuracy is 0.9726\n",
      "after 9900 training steps, the loss is 0.0355115, the validation accuracy is 0.9726\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10000 training steps, the loss is 0.082713, the validation accuracy is 0.9754\n",
      "after 10100 training steps, the loss is 0.0747929, the validation accuracy is 0.976\n",
      "after 10200 training steps, the loss is 0.00387302, the validation accuracy is 0.976\n",
      "after 10300 training steps, the loss is 0.0902695, the validation accuracy is 0.9758\n",
      "after 10400 training steps, the loss is 0.062247, the validation accuracy is 0.976\n",
      "after 10500 training steps, the loss is 0.00319804, the validation accuracy is 0.9774\n",
      "after 10600 training steps, the loss is 0.208292, the validation accuracy is 0.9768\n",
      "after 10700 training steps, the loss is 0.00187122, the validation accuracy is 0.9754\n",
      "after 10800 training steps, the loss is 0.0655172, the validation accuracy is 0.9748\n",
      "after 10900 training steps, the loss is 0.0619849, the validation accuracy is 0.9742\n",
      "after 11000 training steps, the loss is 0.069862, the validation accuracy is 0.974\n",
      "after 11100 training steps, the loss is 0.0037086, the validation accuracy is 0.9782\n",
      "after 11200 training steps, the loss is 0.255067, the validation accuracy is 0.9748\n",
      "after 11300 training steps, the loss is 0.128274, the validation accuracy is 0.975\n",
      "after 11400 training steps, the loss is 0.106319, the validation accuracy is 0.975\n",
      "after 11500 training steps, the loss is 0.0027342, the validation accuracy is 0.975\n",
      "after 11600 training steps, the loss is 0.029454, the validation accuracy is 0.9748\n",
      "after 11700 training steps, the loss is 0.196678, the validation accuracy is 0.9746\n",
      "after 11800 training steps, the loss is 0.00390255, the validation accuracy is 0.9744\n",
      "after 11900 training steps, the loss is 0.0114622, the validation accuracy is 0.9768\n",
      "after 12000 training steps, the loss is 0.0422136, the validation accuracy is 0.973\n",
      "after 12100 training steps, the loss is 0.0114585, the validation accuracy is 0.977\n",
      "after 12200 training steps, the loss is 0.0203142, the validation accuracy is 0.9782\n",
      "after 12300 training steps, the loss is 0.00430615, the validation accuracy is 0.9744\n",
      "after 12400 training steps, the loss is 0.0408038, the validation accuracy is 0.9772\n",
      "after 12500 training steps, the loss is 0.0898486, the validation accuracy is 0.9744\n",
      "after 12600 training steps, the loss is 0.125027, the validation accuracy is 0.9742\n",
      "after 12700 training steps, the loss is 0.0118051, the validation accuracy is 0.9732\n",
      "after 12800 training steps, the loss is 0.0252938, the validation accuracy is 0.9758\n",
      "after 12900 training steps, the loss is 0.0251922, the validation accuracy is 0.9754\n",
      "after 13000 training steps, the loss is 0.0771049, the validation accuracy is 0.9764\n",
      "after 13100 training steps, the loss is 0.00639209, the validation accuracy is 0.976\n",
      "after 13200 training steps, the loss is 0.0039945, the validation accuracy is 0.9786\n",
      "after 13300 training steps, the loss is 0.013788, the validation accuracy is 0.9754\n",
      "after 13400 training steps, the loss is 0.012642, the validation accuracy is 0.9792\n",
      "after 13500 training steps, the loss is 0.00151388, the validation accuracy is 0.9786\n",
      "after 13600 training steps, the loss is 0.0141447, the validation accuracy is 0.9786\n",
      "after 13700 training steps, the loss is 0.0282246, the validation accuracy is 0.9792\n",
      "after 13800 training steps, the loss is 0.00168961, the validation accuracy is 0.9796\n",
      "after 13900 training steps, the loss is 0.340841, the validation accuracy is 0.9778\n",
      "after 14000 training steps, the loss is 0.00404451, the validation accuracy is 0.9756\n",
      "after 14100 training steps, the loss is 0.0305416, the validation accuracy is 0.9766\n",
      "after 14200 training steps, the loss is 0.0123381, the validation accuracy is 0.9764\n",
      "after 14300 training steps, the loss is 0.0932431, the validation accuracy is 0.9758\n",
      "after 14400 training steps, the loss is 0.0191082, the validation accuracy is 0.9768\n",
      "after 14500 training steps, the loss is 0.0073376, the validation accuracy is 0.9782\n",
      "after 14600 training steps, the loss is 0.0191955, the validation accuracy is 0.9776\n",
      "after 14700 training steps, the loss is 0.0110224, the validation accuracy is 0.9786\n",
      "after 14800 training steps, the loss is 0.00347644, the validation accuracy is 0.9782\n",
      "after 14900 training steps, the loss is 0.0143588, the validation accuracy is 0.9776\n",
      "after 15000 training steps, the loss is 0.00358234, the validation accuracy is 0.9764\n",
      "after 15100 training steps, the loss is 0.00302978, the validation accuracy is 0.98\n",
      "after 15200 training steps, the loss is 0.0983618, the validation accuracy is 0.9718\n",
      "after 15300 training steps, the loss is 0.0372629, the validation accuracy is 0.9794\n",
      "after 15400 training steps, the loss is 0.0302152, the validation accuracy is 0.976\n",
      "after 15500 training steps, the loss is 0.00915976, the validation accuracy is 0.9788\n",
      "after 15600 training steps, the loss is 0.00317795, the validation accuracy is 0.9788\n",
      "after 15700 training steps, the loss is 0.00933004, the validation accuracy is 0.9796\n",
      "after 15800 training steps, the loss is 0.0113148, the validation accuracy is 0.9802\n",
      "after 15900 training steps, the loss is 0.00459787, the validation accuracy is 0.977\n",
      "after 16000 training steps, the loss is 0.000980936, the validation accuracy is 0.9774\n",
      "after 16100 training steps, the loss is 0.0264811, the validation accuracy is 0.9766\n",
      "after 16200 training steps, the loss is 0.00121396, the validation accuracy is 0.9762\n",
      "after 16300 training steps, the loss is 0.00297496, the validation accuracy is 0.9776\n",
      "after 16400 training steps, the loss is 0.000482502, the validation accuracy is 0.9778\n",
      "after 16500 training steps, the loss is 0.0102201, the validation accuracy is 0.9778\n",
      "after 16600 training steps, the loss is 0.0036785, the validation accuracy is 0.9794\n",
      "after 16700 training steps, the loss is 0.00795604, the validation accuracy is 0.978\n",
      "after 16800 training steps, the loss is 0.0143833, the validation accuracy is 0.9754\n",
      "after 16900 training steps, the loss is 0.0118286, the validation accuracy is 0.9754\n",
      "after 17000 training steps, the loss is 0.0047508, the validation accuracy is 0.9778\n",
      "after 17100 training steps, the loss is 0.0133453, the validation accuracy is 0.9744\n",
      "after 17200 training steps, the loss is 0.00141696, the validation accuracy is 0.9798\n",
      "after 17300 training steps, the loss is 0.00283104, the validation accuracy is 0.9796\n",
      "after 17400 training steps, the loss is 0.0186068, the validation accuracy is 0.979\n",
      "after 17500 training steps, the loss is 0.0202359, the validation accuracy is 0.976\n",
      "after 17600 training steps, the loss is 0.00510437, the validation accuracy is 0.9782\n",
      "after 17700 training steps, the loss is 0.00334822, the validation accuracy is 0.978\n",
      "after 17800 training steps, the loss is 0.0188874, the validation accuracy is 0.9768\n",
      "after 17900 training steps, the loss is 0.00164347, the validation accuracy is 0.9766\n",
      "after 18000 training steps, the loss is 0.000986265, the validation accuracy is 0.9766\n",
      "after 18100 training steps, the loss is 0.00608244, the validation accuracy is 0.9784\n",
      "after 18200 training steps, the loss is 0.0131895, the validation accuracy is 0.978\n",
      "after 18300 training steps, the loss is 0.00241708, the validation accuracy is 0.9766\n",
      "after 18400 training steps, the loss is 0.00185881, the validation accuracy is 0.9776\n",
      "after 18500 training steps, the loss is 0.00593955, the validation accuracy is 0.9784\n",
      "after 18600 training steps, the loss is 0.00223323, the validation accuracy is 0.9788\n",
      "after 18700 training steps, the loss is 0.0120019, the validation accuracy is 0.9786\n",
      "after 18800 training steps, the loss is 0.00119256, the validation accuracy is 0.978\n",
      "after 18900 training steps, the loss is 0.00105232, the validation accuracy is 0.9778\n",
      "after 19000 training steps, the loss is 0.00682917, the validation accuracy is 0.977\n",
      "after 19100 training steps, the loss is 0.0114774, the validation accuracy is 0.978\n",
      "after 19200 training steps, the loss is 0.000360437, the validation accuracy is 0.9788\n",
      "after 19300 training steps, the loss is 0.0148355, the validation accuracy is 0.9782\n",
      "after 19400 training steps, the loss is 0.00215097, the validation accuracy is 0.978\n",
      "after 19500 training steps, the loss is 0.0143869, the validation accuracy is 0.9788\n",
      "after 19600 training steps, the loss is 0.00217633, the validation accuracy is 0.9776\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19700 training steps, the loss is 0.0133818, the validation accuracy is 0.9776\n",
      "after 19800 training steps, the loss is 0.0132827, the validation accuracy is 0.9792\n",
      "after 19900 training steps, the loss is 0.0153222, the validation accuracy is 0.9784\n",
      "after 20000 training steps, the loss is 0.00670122, the validation accuracy is 0.98\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.9978\n",
      "the test accuarcy is: 0.98\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=20000, lr=0.1, hidden1=100, hidden2=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在增加1个隐藏层的情况下，同样迭代20000次，学习率同样为0.1，模型效果达到了0.98！"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 增加神经元数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.204315, the validation accuracy is 0.886\n",
      "after 200 training steps, the loss is 0.294249, the validation accuracy is 0.919\n",
      "after 300 training steps, the loss is 0.38556, the validation accuracy is 0.9416\n",
      "after 400 training steps, the loss is 0.146709, the validation accuracy is 0.948\n",
      "after 500 training steps, the loss is 0.0399212, the validation accuracy is 0.9532\n",
      "after 600 training steps, the loss is 0.0621882, the validation accuracy is 0.9608\n",
      "after 700 training steps, the loss is 0.182422, the validation accuracy is 0.9586\n",
      "after 800 training steps, the loss is 0.222218, the validation accuracy is 0.9628\n",
      "after 900 training steps, the loss is 0.0380991, the validation accuracy is 0.9624\n",
      "after 1000 training steps, the loss is 0.0141211, the validation accuracy is 0.962\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.96729094\n",
      "the test accuarcy is: 0.9616\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=1000, lr=0.3, hidden1=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "同样的隐藏层数、迭代次数和学习率的情况下，增加神经元数量，模型的准确率从0.95几提升到了0.96几，下面我们增加迭代次数试试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.560837, the validation accuracy is 0.8922\n",
      "after 200 training steps, the loss is 0.400309, the validation accuracy is 0.9226\n",
      "after 300 training steps, the loss is 0.501165, the validation accuracy is 0.93\n",
      "after 400 training steps, the loss is 0.154623, the validation accuracy is 0.9364\n",
      "after 500 training steps, the loss is 0.366229, the validation accuracy is 0.9302\n",
      "after 600 training steps, the loss is 0.127047, the validation accuracy is 0.9418\n",
      "after 700 training steps, the loss is 0.187716, the validation accuracy is 0.9442\n",
      "after 800 training steps, the loss is 0.126355, the validation accuracy is 0.9504\n",
      "after 900 training steps, the loss is 0.0992885, the validation accuracy is 0.954\n",
      "after 1000 training steps, the loss is 0.1291, the validation accuracy is 0.9564\n",
      "after 1100 training steps, the loss is 0.211152, the validation accuracy is 0.9582\n",
      "after 1200 training steps, the loss is 0.179248, the validation accuracy is 0.9548\n",
      "after 1300 training steps, the loss is 0.127317, the validation accuracy is 0.9586\n",
      "after 1400 training steps, the loss is 0.0266807, the validation accuracy is 0.9614\n",
      "after 1500 training steps, the loss is 0.0236781, the validation accuracy is 0.9538\n",
      "after 1600 training steps, the loss is 0.0462794, the validation accuracy is 0.9598\n",
      "after 1700 training steps, the loss is 0.0746713, the validation accuracy is 0.9594\n",
      "after 1800 training steps, the loss is 0.163957, the validation accuracy is 0.961\n",
      "after 1900 training steps, the loss is 0.0912887, the validation accuracy is 0.9646\n",
      "after 2000 training steps, the loss is 0.0948568, the validation accuracy is 0.962\n",
      "after 2100 training steps, the loss is 0.212119, the validation accuracy is 0.9702\n",
      "after 2200 training steps, the loss is 0.0431803, the validation accuracy is 0.969\n",
      "after 2300 training steps, the loss is 0.105829, the validation accuracy is 0.9678\n",
      "after 2400 training steps, the loss is 0.117693, the validation accuracy is 0.968\n",
      "after 2500 training steps, the loss is 0.233324, the validation accuracy is 0.9646\n",
      "after 2600 training steps, the loss is 0.0958813, the validation accuracy is 0.9672\n",
      "after 2700 training steps, the loss is 0.211934, the validation accuracy is 0.9718\n",
      "after 2800 training steps, the loss is 0.169163, the validation accuracy is 0.9682\n",
      "after 2900 training steps, the loss is 0.0439183, the validation accuracy is 0.9698\n",
      "after 3000 training steps, the loss is 0.0296412, the validation accuracy is 0.9706\n",
      "after 3100 training steps, the loss is 0.137553, the validation accuracy is 0.9706\n",
      "after 3200 training steps, the loss is 0.0355073, the validation accuracy is 0.9724\n",
      "after 3300 training steps, the loss is 0.0233579, the validation accuracy is 0.969\n",
      "after 3400 training steps, the loss is 0.0312657, the validation accuracy is 0.968\n",
      "after 3500 training steps, the loss is 0.0518849, the validation accuracy is 0.9734\n",
      "after 3600 training steps, the loss is 0.0352424, the validation accuracy is 0.9734\n",
      "after 3700 training steps, the loss is 0.0883404, the validation accuracy is 0.9716\n",
      "after 3800 training steps, the loss is 0.0509622, the validation accuracy is 0.9746\n",
      "after 3900 training steps, the loss is 0.0949731, the validation accuracy is 0.9722\n",
      "after 4000 training steps, the loss is 0.0248075, the validation accuracy is 0.972\n",
      "after 4100 training steps, the loss is 0.0787239, the validation accuracy is 0.9738\n",
      "after 4200 training steps, the loss is 0.0490342, the validation accuracy is 0.9738\n",
      "after 4300 training steps, the loss is 0.0892314, the validation accuracy is 0.9744\n",
      "after 4400 training steps, the loss is 0.132459, the validation accuracy is 0.9736\n",
      "after 4500 training steps, the loss is 0.0325573, the validation accuracy is 0.9724\n",
      "after 4600 training steps, the loss is 0.0447057, the validation accuracy is 0.9754\n",
      "after 4700 training steps, the loss is 0.0583554, the validation accuracy is 0.9748\n",
      "after 4800 training steps, the loss is 0.0445485, the validation accuracy is 0.975\n",
      "after 4900 training steps, the loss is 0.0314547, the validation accuracy is 0.9766\n",
      "after 5000 training steps, the loss is 0.0613379, the validation accuracy is 0.9726\n",
      "after 5100 training steps, the loss is 0.186777, the validation accuracy is 0.9734\n",
      "after 5200 training steps, the loss is 0.0616721, the validation accuracy is 0.9754\n",
      "after 5300 training steps, the loss is 0.0426992, the validation accuracy is 0.9764\n",
      "after 5400 training steps, the loss is 0.0076791, the validation accuracy is 0.9724\n",
      "after 5500 training steps, the loss is 0.0271318, the validation accuracy is 0.9722\n",
      "after 5600 training steps, the loss is 0.0172216, the validation accuracy is 0.9746\n",
      "after 5700 training steps, the loss is 0.0478623, the validation accuracy is 0.9744\n",
      "after 5800 training steps, the loss is 0.0121293, the validation accuracy is 0.9744\n",
      "after 5900 training steps, the loss is 0.107329, the validation accuracy is 0.975\n",
      "after 6000 training steps, the loss is 0.0557177, the validation accuracy is 0.9766\n",
      "after 6100 training steps, the loss is 0.0929958, the validation accuracy is 0.9754\n",
      "after 6200 training steps, the loss is 0.0192279, the validation accuracy is 0.9772\n",
      "after 6300 training steps, the loss is 0.0280426, the validation accuracy is 0.978\n",
      "after 6400 training steps, the loss is 0.0792485, the validation accuracy is 0.9784\n",
      "after 6500 training steps, the loss is 0.0829993, the validation accuracy is 0.9774\n",
      "after 6600 training steps, the loss is 0.0166708, the validation accuracy is 0.9766\n",
      "after 6700 training steps, the loss is 0.012058, the validation accuracy is 0.9772\n",
      "after 6800 training steps, the loss is 0.00945931, the validation accuracy is 0.977\n",
      "after 6900 training steps, the loss is 0.113691, the validation accuracy is 0.9766\n",
      "after 7000 training steps, the loss is 0.0593908, the validation accuracy is 0.9778\n",
      "after 7100 training steps, the loss is 0.0509757, the validation accuracy is 0.9766\n",
      "after 7200 training steps, the loss is 0.0900858, the validation accuracy is 0.9758\n",
      "after 7300 training steps, the loss is 0.173179, the validation accuracy is 0.9766\n",
      "after 7400 training steps, the loss is 0.00812633, the validation accuracy is 0.9786\n",
      "after 7500 training steps, the loss is 0.0692733, the validation accuracy is 0.979\n",
      "after 7600 training steps, the loss is 0.012018, the validation accuracy is 0.9792\n",
      "after 7700 training steps, the loss is 0.030247, the validation accuracy is 0.9764\n",
      "after 7800 training steps, the loss is 0.0473067, the validation accuracy is 0.9786\n",
      "after 7900 training steps, the loss is 0.0141594, the validation accuracy is 0.9782\n",
      "after 8000 training steps, the loss is 0.0204732, the validation accuracy is 0.978\n",
      "after 8100 training steps, the loss is 0.0115186, the validation accuracy is 0.9784\n",
      "after 8200 training steps, the loss is 0.0456529, the validation accuracy is 0.9768\n",
      "after 8300 training steps, the loss is 0.0251829, the validation accuracy is 0.978\n",
      "after 8400 training steps, the loss is 0.0339451, the validation accuracy is 0.9774\n",
      "after 8500 training steps, the loss is 0.0154022, the validation accuracy is 0.977\n",
      "after 8600 training steps, the loss is 0.0281625, the validation accuracy is 0.9774\n",
      "after 8700 training steps, the loss is 0.0227438, the validation accuracy is 0.9782\n",
      "after 8800 training steps, the loss is 0.0651086, the validation accuracy is 0.9792\n",
      "after 8900 training steps, the loss is 0.0160794, the validation accuracy is 0.978\n",
      "after 9000 training steps, the loss is 0.0658051, the validation accuracy is 0.979\n",
      "after 9100 training steps, the loss is 0.00665825, the validation accuracy is 0.9786\n",
      "after 9200 training steps, the loss is 0.0192005, the validation accuracy is 0.9806\n",
      "after 9300 training steps, the loss is 0.0104118, the validation accuracy is 0.9788\n",
      "after 9400 training steps, the loss is 0.0271518, the validation accuracy is 0.9788\n",
      "after 9500 training steps, the loss is 0.0055159, the validation accuracy is 0.9778\n",
      "after 9600 training steps, the loss is 0.0123224, the validation accuracy is 0.9796\n",
      "after 9700 training steps, the loss is 0.0145385, the validation accuracy is 0.9788\n",
      "after 9800 training steps, the loss is 0.0251635, the validation accuracy is 0.979\n",
      "after 9900 training steps, the loss is 0.0942426, the validation accuracy is 0.9794\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10000 training steps, the loss is 0.0104066, the validation accuracy is 0.9794\n",
      "after 10100 training steps, the loss is 0.00795249, the validation accuracy is 0.9806\n",
      "after 10200 training steps, the loss is 0.0188382, the validation accuracy is 0.9786\n",
      "after 10300 training steps, the loss is 0.0249876, the validation accuracy is 0.9784\n",
      "after 10400 training steps, the loss is 0.0170965, the validation accuracy is 0.977\n",
      "after 10500 training steps, the loss is 0.0151895, the validation accuracy is 0.9806\n",
      "after 10600 training steps, the loss is 0.0177361, the validation accuracy is 0.9816\n",
      "after 10700 training steps, the loss is 0.0121043, the validation accuracy is 0.9776\n",
      "after 10800 training steps, the loss is 0.00710977, the validation accuracy is 0.9794\n",
      "after 10900 training steps, the loss is 0.0389205, the validation accuracy is 0.9798\n",
      "after 11000 training steps, the loss is 0.00513706, the validation accuracy is 0.9794\n",
      "after 11100 training steps, the loss is 0.0493854, the validation accuracy is 0.978\n",
      "after 11200 training steps, the loss is 0.0123237, the validation accuracy is 0.9806\n",
      "after 11300 training steps, the loss is 0.0140933, the validation accuracy is 0.9798\n",
      "after 11400 training steps, the loss is 0.0053548, the validation accuracy is 0.9804\n",
      "after 11500 training steps, the loss is 0.0303455, the validation accuracy is 0.9808\n",
      "after 11600 training steps, the loss is 0.000535609, the validation accuracy is 0.9798\n",
      "after 11700 training steps, the loss is 0.0129696, the validation accuracy is 0.9798\n",
      "after 11800 training steps, the loss is 0.0531105, the validation accuracy is 0.9778\n",
      "after 11900 training steps, the loss is 0.00377187, the validation accuracy is 0.9778\n",
      "after 12000 training steps, the loss is 0.0606627, the validation accuracy is 0.981\n",
      "after 12100 training steps, the loss is 0.00447826, the validation accuracy is 0.9796\n",
      "after 12200 training steps, the loss is 0.0335061, the validation accuracy is 0.9814\n",
      "after 12300 training steps, the loss is 0.0417564, the validation accuracy is 0.9784\n",
      "after 12400 training steps, the loss is 0.0286474, the validation accuracy is 0.9818\n",
      "after 12500 training steps, the loss is 0.0139143, the validation accuracy is 0.9812\n",
      "after 12600 training steps, the loss is 0.0118881, the validation accuracy is 0.98\n",
      "after 12700 training steps, the loss is 0.00694423, the validation accuracy is 0.9796\n",
      "after 12800 training steps, the loss is 0.0353759, the validation accuracy is 0.9814\n",
      "after 12900 training steps, the loss is 0.00422306, the validation accuracy is 0.9824\n",
      "after 13000 training steps, the loss is 0.00522104, the validation accuracy is 0.9818\n",
      "after 13100 training steps, the loss is 0.00561467, the validation accuracy is 0.9816\n",
      "after 13200 training steps, the loss is 0.0394495, the validation accuracy is 0.9812\n",
      "after 13300 training steps, the loss is 0.0374962, the validation accuracy is 0.9814\n",
      "after 13400 training steps, the loss is 0.00963766, the validation accuracy is 0.9814\n",
      "after 13500 training steps, the loss is 0.0929053, the validation accuracy is 0.9814\n",
      "after 13600 training steps, the loss is 0.00838383, the validation accuracy is 0.9802\n",
      "after 13700 training steps, the loss is 0.0306722, the validation accuracy is 0.9812\n",
      "after 13800 training steps, the loss is 0.00445099, the validation accuracy is 0.9818\n",
      "after 13900 training steps, the loss is 0.0110058, the validation accuracy is 0.9814\n",
      "after 14000 training steps, the loss is 0.0100945, the validation accuracy is 0.9814\n",
      "after 14100 training steps, the loss is 0.0098536, the validation accuracy is 0.9812\n",
      "after 14200 training steps, the loss is 0.0729769, the validation accuracy is 0.9798\n",
      "after 14300 training steps, the loss is 0.0496328, the validation accuracy is 0.9798\n",
      "after 14400 training steps, the loss is 0.0115815, the validation accuracy is 0.9804\n",
      "after 14500 training steps, the loss is 0.00368693, the validation accuracy is 0.9818\n",
      "after 14600 training steps, the loss is 0.0190355, the validation accuracy is 0.9808\n",
      "after 14700 training steps, the loss is 0.016955, the validation accuracy is 0.9802\n",
      "after 14800 training steps, the loss is 0.144013, the validation accuracy is 0.9804\n",
      "after 14900 training steps, the loss is 0.0101296, the validation accuracy is 0.9806\n",
      "after 15000 training steps, the loss is 0.00477359, the validation accuracy is 0.9824\n",
      "after 15100 training steps, the loss is 0.0117252, the validation accuracy is 0.9812\n",
      "after 15200 training steps, the loss is 0.0069043, the validation accuracy is 0.9816\n",
      "after 15300 training steps, the loss is 0.00382741, the validation accuracy is 0.9802\n",
      "after 15400 training steps, the loss is 0.00825322, the validation accuracy is 0.9816\n",
      "after 15500 training steps, the loss is 0.045325, the validation accuracy is 0.982\n",
      "after 15600 training steps, the loss is 0.0432035, the validation accuracy is 0.9812\n",
      "after 15700 training steps, the loss is 0.013911, the validation accuracy is 0.9812\n",
      "after 15800 training steps, the loss is 0.0124864, the validation accuracy is 0.9818\n",
      "after 15900 training steps, the loss is 0.00404898, the validation accuracy is 0.9816\n",
      "after 16000 training steps, the loss is 0.0118504, the validation accuracy is 0.9816\n",
      "after 16100 training steps, the loss is 0.00469318, the validation accuracy is 0.9808\n",
      "after 16200 training steps, the loss is 0.0150548, the validation accuracy is 0.9814\n",
      "after 16300 training steps, the loss is 0.0168127, the validation accuracy is 0.9816\n",
      "after 16400 training steps, the loss is 0.00960674, the validation accuracy is 0.9816\n",
      "after 16500 training steps, the loss is 0.00725252, the validation accuracy is 0.9818\n",
      "after 16600 training steps, the loss is 0.0025741, the validation accuracy is 0.9822\n",
      "after 16700 training steps, the loss is 0.00715879, the validation accuracy is 0.9828\n",
      "after 16800 training steps, the loss is 0.0110707, the validation accuracy is 0.982\n",
      "after 16900 training steps, the loss is 0.0294462, the validation accuracy is 0.981\n",
      "after 17000 training steps, the loss is 0.00416896, the validation accuracy is 0.9816\n",
      "after 17100 training steps, the loss is 0.00303029, the validation accuracy is 0.9812\n",
      "after 17200 training steps, the loss is 0.0108596, the validation accuracy is 0.9818\n",
      "after 17300 training steps, the loss is 0.0167241, the validation accuracy is 0.9818\n",
      "after 17400 training steps, the loss is 0.00970815, the validation accuracy is 0.9806\n",
      "after 17500 training steps, the loss is 0.00386938, the validation accuracy is 0.9806\n",
      "after 17600 training steps, the loss is 0.000965871, the validation accuracy is 0.9826\n",
      "after 17700 training steps, the loss is 0.0201362, the validation accuracy is 0.9824\n",
      "after 17800 training steps, the loss is 0.00122448, the validation accuracy is 0.9806\n",
      "after 17900 training steps, the loss is 0.0267979, the validation accuracy is 0.9822\n",
      "after 18000 training steps, the loss is 0.00878228, the validation accuracy is 0.983\n",
      "after 18100 training steps, the loss is 0.00701963, the validation accuracy is 0.9814\n",
      "after 18200 training steps, the loss is 0.00459019, the validation accuracy is 0.9828\n",
      "after 18300 training steps, the loss is 0.00174676, the validation accuracy is 0.981\n",
      "after 18400 training steps, the loss is 0.00683868, the validation accuracy is 0.9808\n",
      "after 18500 training steps, the loss is 0.00360958, the validation accuracy is 0.9814\n",
      "after 18600 training steps, the loss is 0.00410951, the validation accuracy is 0.9814\n",
      "after 18700 training steps, the loss is 0.00649983, the validation accuracy is 0.9812\n",
      "after 18800 training steps, the loss is 0.0177208, the validation accuracy is 0.9814\n",
      "after 18900 training steps, the loss is 0.0152973, the validation accuracy is 0.982\n",
      "after 19000 training steps, the loss is 0.00938898, the validation accuracy is 0.9812\n",
      "after 19100 training steps, the loss is 0.00650232, the validation accuracy is 0.9822\n",
      "after 19200 training steps, the loss is 0.00518025, the validation accuracy is 0.9798\n",
      "after 19300 training steps, the loss is 0.000908099, the validation accuracy is 0.982\n",
      "after 19400 training steps, the loss is 0.00115452, the validation accuracy is 0.982\n",
      "after 19500 training steps, the loss is 0.00136232, the validation accuracy is 0.983\n",
      "after 19600 training steps, the loss is 0.00669791, the validation accuracy is 0.9836\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19700 training steps, the loss is 0.0184933, the validation accuracy is 0.9814\n",
      "after 19800 training steps, the loss is 0.00477739, the validation accuracy is 0.9828\n",
      "after 19900 training steps, the loss is 0.0049122, the validation accuracy is 0.982\n",
      "after 20000 training steps, the loss is 0.010337, the validation accuracy is 0.9828\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.99972725\n",
      "the test accuarcy is: 0.9807\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=20000, lr=0.1, hidden1=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将迭代次数同样调到20000次，学习率为0.1，模型准确率达到了0.98以上！！"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从上面的结果我们发现在训练集上的准确率达到了0.999，测试集上准确率才刚刚过0.98，存在过拟合问题，接下来我们可以试试添加正则化参数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 添加L1/L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.899265, the validation accuracy is 0.8868\n",
      "after 200 training steps, the loss is 0.872033, the validation accuracy is 0.9118\n",
      "after 300 training steps, the loss is 1.01907, the validation accuracy is 0.9242\n",
      "after 400 training steps, the loss is 0.552514, the validation accuracy is 0.935\n",
      "after 500 training steps, the loss is 0.767209, the validation accuracy is 0.9348\n",
      "after 600 training steps, the loss is 0.713169, the validation accuracy is 0.9274\n",
      "after 700 training steps, the loss is 0.737643, the validation accuracy is 0.94\n",
      "after 800 training steps, the loss is 0.595472, the validation accuracy is 0.9406\n",
      "after 900 training steps, the loss is 0.548893, the validation accuracy is 0.9372\n",
      "after 1000 training steps, the loss is 0.427116, the validation accuracy is 0.9448\n",
      "after 1100 training steps, the loss is 0.45663, the validation accuracy is 0.9406\n",
      "after 1200 training steps, the loss is 0.4798, the validation accuracy is 0.9454\n",
      "after 1300 training steps, the loss is 0.430437, the validation accuracy is 0.9442\n",
      "after 1400 training steps, the loss is 0.2953, the validation accuracy is 0.945\n",
      "after 1500 training steps, the loss is 0.315117, the validation accuracy is 0.9412\n",
      "after 1600 training steps, the loss is 0.447277, the validation accuracy is 0.932\n",
      "after 1700 training steps, the loss is 0.560558, the validation accuracy is 0.944\n",
      "after 1800 training steps, the loss is 0.40223, the validation accuracy is 0.9442\n",
      "after 1900 training steps, the loss is 0.320904, the validation accuracy is 0.9382\n",
      "after 2000 training steps, the loss is 0.483892, the validation accuracy is 0.9514\n",
      "after 2100 training steps, the loss is 0.308973, the validation accuracy is 0.9502\n",
      "after 2200 training steps, the loss is 0.376038, the validation accuracy is 0.9486\n",
      "after 2300 training steps, the loss is 0.365084, the validation accuracy is 0.9522\n",
      "after 2400 training steps, the loss is 0.351179, the validation accuracy is 0.948\n",
      "after 2500 training steps, the loss is 0.480133, the validation accuracy is 0.9504\n",
      "after 2600 training steps, the loss is 0.34915, the validation accuracy is 0.9498\n",
      "after 2700 training steps, the loss is 0.378934, the validation accuracy is 0.9554\n",
      "after 2800 training steps, the loss is 0.282058, the validation accuracy is 0.955\n",
      "after 2900 training steps, the loss is 0.291175, the validation accuracy is 0.9448\n",
      "after 3000 training steps, the loss is 0.199743, the validation accuracy is 0.9554\n",
      "after 3100 training steps, the loss is 0.214194, the validation accuracy is 0.9554\n",
      "after 3200 training steps, the loss is 0.267972, the validation accuracy is 0.9558\n",
      "after 3300 training steps, the loss is 0.19573, the validation accuracy is 0.9548\n",
      "after 3400 training steps, the loss is 0.250901, the validation accuracy is 0.9548\n",
      "after 3500 training steps, the loss is 0.389446, the validation accuracy is 0.951\n",
      "after 3600 training steps, the loss is 0.648643, the validation accuracy is 0.96\n",
      "after 3700 training steps, the loss is 0.203058, the validation accuracy is 0.9604\n",
      "after 3800 training steps, the loss is 0.216781, the validation accuracy is 0.9572\n",
      "after 3900 training steps, the loss is 0.223902, the validation accuracy is 0.9566\n",
      "after 4000 training steps, the loss is 0.153366, the validation accuracy is 0.959\n",
      "after 4100 training steps, the loss is 0.344447, the validation accuracy is 0.9574\n",
      "after 4200 training steps, the loss is 0.232964, the validation accuracy is 0.9556\n",
      "after 4300 training steps, the loss is 0.214718, the validation accuracy is 0.9602\n",
      "after 4400 training steps, the loss is 0.194693, the validation accuracy is 0.9564\n",
      "after 4500 training steps, the loss is 0.240885, the validation accuracy is 0.963\n",
      "after 4600 training steps, the loss is 0.298765, the validation accuracy is 0.9586\n",
      "after 4700 training steps, the loss is 0.198301, the validation accuracy is 0.9554\n",
      "after 4800 training steps, the loss is 0.155055, the validation accuracy is 0.9618\n",
      "after 4900 training steps, the loss is 0.240993, the validation accuracy is 0.9562\n",
      "after 5000 training steps, the loss is 0.321638, the validation accuracy is 0.9586\n",
      "after 5100 training steps, the loss is 0.258777, the validation accuracy is 0.9606\n",
      "after 5200 training steps, the loss is 0.32622, the validation accuracy is 0.958\n",
      "after 5300 training steps, the loss is 0.196018, the validation accuracy is 0.961\n",
      "after 5400 training steps, the loss is 0.311055, the validation accuracy is 0.9602\n",
      "after 5500 training steps, the loss is 0.233804, the validation accuracy is 0.9598\n",
      "after 5600 training steps, the loss is 0.420122, the validation accuracy is 0.9632\n",
      "after 5700 training steps, the loss is 0.16961, the validation accuracy is 0.9624\n",
      "after 5800 training steps, the loss is 0.3233, the validation accuracy is 0.9606\n",
      "after 5900 training steps, the loss is 0.23679, the validation accuracy is 0.9614\n",
      "after 6000 training steps, the loss is 0.244376, the validation accuracy is 0.9588\n",
      "after 6100 training steps, the loss is 0.273063, the validation accuracy is 0.965\n",
      "after 6200 training steps, the loss is 0.145255, the validation accuracy is 0.962\n",
      "after 6300 training steps, the loss is 0.14969, the validation accuracy is 0.963\n",
      "after 6400 training steps, the loss is 0.240736, the validation accuracy is 0.96\n",
      "after 6500 training steps, the loss is 0.172193, the validation accuracy is 0.9628\n",
      "after 6600 training steps, the loss is 0.249987, the validation accuracy is 0.9642\n",
      "after 6700 training steps, the loss is 0.22342, the validation accuracy is 0.963\n",
      "after 6800 training steps, the loss is 0.241964, the validation accuracy is 0.9628\n",
      "after 6900 training steps, the loss is 0.175703, the validation accuracy is 0.9648\n",
      "after 7000 training steps, the loss is 0.177219, the validation accuracy is 0.964\n",
      "after 7100 training steps, the loss is 0.142971, the validation accuracy is 0.9672\n",
      "after 7200 training steps, the loss is 0.271601, the validation accuracy is 0.9646\n",
      "after 7300 training steps, the loss is 0.158736, the validation accuracy is 0.9612\n",
      "after 7400 training steps, the loss is 0.138404, the validation accuracy is 0.9646\n",
      "after 7500 training steps, the loss is 0.193931, the validation accuracy is 0.9658\n",
      "after 7600 training steps, the loss is 0.150741, the validation accuracy is 0.9636\n",
      "after 7700 training steps, the loss is 0.175139, the validation accuracy is 0.9674\n",
      "after 7800 training steps, the loss is 0.161486, the validation accuracy is 0.959\n",
      "after 7900 training steps, the loss is 0.176507, the validation accuracy is 0.959\n",
      "after 8000 training steps, the loss is 0.190034, the validation accuracy is 0.9646\n",
      "after 8100 training steps, the loss is 0.169553, the validation accuracy is 0.9668\n",
      "after 8200 training steps, the loss is 0.229251, the validation accuracy is 0.9672\n",
      "after 8300 training steps, the loss is 0.190397, the validation accuracy is 0.9652\n",
      "after 8400 training steps, the loss is 0.191456, the validation accuracy is 0.9642\n",
      "after 8500 training steps, the loss is 0.195265, the validation accuracy is 0.9648\n",
      "after 8600 training steps, the loss is 0.217788, the validation accuracy is 0.9656\n",
      "after 8700 training steps, the loss is 0.129643, the validation accuracy is 0.9662\n",
      "after 8800 training steps, the loss is 0.263929, the validation accuracy is 0.9676\n",
      "after 8900 training steps, the loss is 0.194268, the validation accuracy is 0.965\n",
      "after 9000 training steps, the loss is 0.199007, the validation accuracy is 0.9656\n",
      "after 9100 training steps, the loss is 0.457787, the validation accuracy is 0.9638\n",
      "after 9200 training steps, the loss is 0.507459, the validation accuracy is 0.9648\n",
      "after 9300 training steps, the loss is 0.163028, the validation accuracy is 0.9602\n",
      "after 9400 training steps, the loss is 0.143173, the validation accuracy is 0.964\n",
      "after 9500 training steps, the loss is 0.13386, the validation accuracy is 0.9698\n",
      "after 9600 training steps, the loss is 0.141677, the validation accuracy is 0.9672\n",
      "after 9700 training steps, the loss is 0.329086, the validation accuracy is 0.9686\n",
      "after 9800 training steps, the loss is 0.185118, the validation accuracy is 0.9682\n",
      "after 9900 training steps, the loss is 0.14093, the validation accuracy is 0.967\n",
      "after 10000 training steps, the loss is 0.296936, the validation accuracy is 0.9622\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10100 training steps, the loss is 0.239682, the validation accuracy is 0.9668\n",
      "after 10200 training steps, the loss is 0.283429, the validation accuracy is 0.9658\n",
      "after 10300 training steps, the loss is 0.280955, the validation accuracy is 0.9656\n",
      "after 10400 training steps, the loss is 0.154075, the validation accuracy is 0.9646\n",
      "after 10500 training steps, the loss is 0.12332, the validation accuracy is 0.9648\n",
      "after 10600 training steps, the loss is 0.191311, the validation accuracy is 0.967\n",
      "after 10700 training steps, the loss is 0.299994, the validation accuracy is 0.9678\n",
      "after 10800 training steps, the loss is 0.337646, the validation accuracy is 0.97\n",
      "after 10900 training steps, the loss is 0.193674, the validation accuracy is 0.9668\n",
      "after 11000 training steps, the loss is 0.301889, the validation accuracy is 0.9682\n",
      "after 11100 training steps, the loss is 0.146493, the validation accuracy is 0.9682\n",
      "after 11200 training steps, the loss is 0.21976, the validation accuracy is 0.9688\n",
      "after 11300 training steps, the loss is 0.145548, the validation accuracy is 0.9688\n",
      "after 11400 training steps, the loss is 0.120877, the validation accuracy is 0.9668\n",
      "after 11500 training steps, the loss is 0.181304, the validation accuracy is 0.9692\n",
      "after 11600 training steps, the loss is 0.137136, the validation accuracy is 0.9694\n",
      "after 11700 training steps, the loss is 0.142004, the validation accuracy is 0.9654\n",
      "after 11800 training steps, the loss is 0.192321, the validation accuracy is 0.968\n",
      "after 11900 training steps, the loss is 0.248027, the validation accuracy is 0.9688\n",
      "after 12000 training steps, the loss is 0.149026, the validation accuracy is 0.9688\n",
      "after 12100 training steps, the loss is 0.273029, the validation accuracy is 0.9702\n",
      "after 12200 training steps, the loss is 0.248945, the validation accuracy is 0.9672\n",
      "after 12300 training steps, the loss is 0.146426, the validation accuracy is 0.968\n",
      "after 12400 training steps, the loss is 0.168863, the validation accuracy is 0.9696\n",
      "after 12500 training steps, the loss is 0.282605, the validation accuracy is 0.968\n",
      "after 12600 training steps, the loss is 0.105166, the validation accuracy is 0.9702\n",
      "after 12700 training steps, the loss is 0.271739, the validation accuracy is 0.9674\n",
      "after 12800 training steps, the loss is 0.193239, the validation accuracy is 0.9692\n",
      "after 12900 training steps, the loss is 0.146397, the validation accuracy is 0.9672\n",
      "after 13000 training steps, the loss is 0.117558, the validation accuracy is 0.9678\n",
      "after 13100 training steps, the loss is 0.202159, the validation accuracy is 0.9682\n",
      "after 13200 training steps, the loss is 0.143084, the validation accuracy is 0.9666\n",
      "after 13300 training steps, the loss is 0.138816, the validation accuracy is 0.966\n",
      "after 13400 training steps, the loss is 0.13914, the validation accuracy is 0.9674\n",
      "after 13500 training steps, the loss is 0.163667, the validation accuracy is 0.9684\n",
      "after 13600 training steps, the loss is 0.154037, the validation accuracy is 0.9696\n",
      "after 13700 training steps, the loss is 0.138361, the validation accuracy is 0.9698\n",
      "after 13800 training steps, the loss is 0.152875, the validation accuracy is 0.9698\n",
      "after 13900 training steps, the loss is 0.119726, the validation accuracy is 0.9696\n",
      "after 14000 training steps, the loss is 0.10682, the validation accuracy is 0.968\n",
      "after 14100 training steps, the loss is 0.101134, the validation accuracy is 0.9688\n",
      "after 14200 training steps, the loss is 0.247305, the validation accuracy is 0.9684\n",
      "after 14300 training steps, the loss is 0.128244, the validation accuracy is 0.971\n",
      "after 14400 training steps, the loss is 0.121604, the validation accuracy is 0.9688\n",
      "after 14500 training steps, the loss is 0.145113, the validation accuracy is 0.969\n",
      "after 14600 training steps, the loss is 0.110959, the validation accuracy is 0.9654\n",
      "after 14700 training steps, the loss is 0.242384, the validation accuracy is 0.9672\n",
      "after 14800 training steps, the loss is 0.209542, the validation accuracy is 0.9698\n",
      "after 14900 training steps, the loss is 0.120595, the validation accuracy is 0.9688\n",
      "after 15000 training steps, the loss is 0.117824, the validation accuracy is 0.967\n",
      "after 15100 training steps, the loss is 0.158323, the validation accuracy is 0.9666\n",
      "after 15200 training steps, the loss is 0.135187, the validation accuracy is 0.9718\n",
      "after 15300 training steps, the loss is 0.259133, the validation accuracy is 0.9694\n",
      "after 15400 training steps, the loss is 0.129411, the validation accuracy is 0.9704\n",
      "after 15500 training steps, the loss is 0.165867, the validation accuracy is 0.9714\n",
      "after 15600 training steps, the loss is 0.25838, the validation accuracy is 0.97\n",
      "after 15700 training steps, the loss is 0.20234, the validation accuracy is 0.966\n",
      "after 15800 training steps, the loss is 0.123589, the validation accuracy is 0.9676\n",
      "after 15900 training steps, the loss is 0.111223, the validation accuracy is 0.9708\n",
      "after 16000 training steps, the loss is 0.126455, the validation accuracy is 0.9706\n",
      "after 16100 training steps, the loss is 0.370968, the validation accuracy is 0.9672\n",
      "after 16200 training steps, the loss is 0.183902, the validation accuracy is 0.9698\n",
      "after 16300 training steps, the loss is 0.115216, the validation accuracy is 0.9698\n",
      "after 16400 training steps, the loss is 0.137509, the validation accuracy is 0.9694\n",
      "after 16500 training steps, the loss is 0.259809, the validation accuracy is 0.9704\n",
      "after 16600 training steps, the loss is 0.153406, the validation accuracy is 0.9712\n",
      "after 16700 training steps, the loss is 0.104101, the validation accuracy is 0.9714\n",
      "after 16800 training steps, the loss is 0.113268, the validation accuracy is 0.9704\n",
      "after 16900 training steps, the loss is 0.108679, the validation accuracy is 0.9694\n",
      "after 17000 training steps, the loss is 0.169857, the validation accuracy is 0.9682\n",
      "after 17100 training steps, the loss is 0.151291, the validation accuracy is 0.9704\n",
      "after 17200 training steps, the loss is 0.235134, the validation accuracy is 0.97\n",
      "after 17300 training steps, the loss is 0.277058, the validation accuracy is 0.9692\n",
      "after 17400 training steps, the loss is 0.19246, the validation accuracy is 0.9706\n",
      "after 17500 training steps, the loss is 0.204691, the validation accuracy is 0.9726\n",
      "after 17600 training steps, the loss is 0.375259, the validation accuracy is 0.9696\n",
      "after 17700 training steps, the loss is 0.165497, the validation accuracy is 0.9688\n",
      "after 17800 training steps, the loss is 0.139841, the validation accuracy is 0.9696\n",
      "after 17900 training steps, the loss is 0.24351, the validation accuracy is 0.969\n",
      "after 18000 training steps, the loss is 0.150773, the validation accuracy is 0.9696\n",
      "after 18100 training steps, the loss is 0.125647, the validation accuracy is 0.9722\n",
      "after 18200 training steps, the loss is 0.0938744, the validation accuracy is 0.9702\n",
      "after 18300 training steps, the loss is 0.204494, the validation accuracy is 0.9708\n",
      "after 18400 training steps, the loss is 0.103766, the validation accuracy is 0.9706\n",
      "after 18500 training steps, the loss is 0.183328, the validation accuracy is 0.9718\n",
      "after 18600 training steps, the loss is 0.117132, the validation accuracy is 0.9672\n",
      "after 18700 training steps, the loss is 0.102149, the validation accuracy is 0.9712\n",
      "after 18800 training steps, the loss is 0.231439, the validation accuracy is 0.9704\n",
      "after 18900 training steps, the loss is 0.159175, the validation accuracy is 0.9688\n",
      "after 19000 training steps, the loss is 0.105078, the validation accuracy is 0.9716\n",
      "after 19100 training steps, the loss is 0.191877, the validation accuracy is 0.9694\n",
      "after 19200 training steps, the loss is 0.237855, the validation accuracy is 0.9722\n",
      "after 19300 training steps, the loss is 0.17112, the validation accuracy is 0.9706\n",
      "after 19400 training steps, the loss is 0.10648, the validation accuracy is 0.9718\n",
      "after 19500 training steps, the loss is 0.0916191, the validation accuracy is 0.9726\n",
      "after 19600 training steps, the loss is 0.148958, the validation accuracy is 0.9712\n",
      "after 19700 training steps, the loss is 0.186761, the validation accuracy is 0.9702\n",
      "after 19800 training steps, the loss is 0.100648, the validation accuracy is 0.973\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19900 training steps, the loss is 0.130681, the validation accuracy is 0.9714\n",
      "after 20000 training steps, the loss is 0.111942, the validation accuracy is 0.9708\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.97865456\n",
      "the test accuarcy is: 0.969\n"
     ]
    }
   ],
   "source": [
    "# l1正则\n",
    "train(batch_size=32, training_step=20000, lr=0.1, hidden1=1000, lambda_flag='l1', lambda_value=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.630872, the validation accuracy is 0.8852\n",
      "after 200 training steps, the loss is 0.333897, the validation accuracy is 0.91\n",
      "after 300 training steps, the loss is 0.527852, the validation accuracy is 0.92\n",
      "after 400 training steps, the loss is 0.6189, the validation accuracy is 0.8888\n",
      "after 500 training steps, the loss is 0.437061, the validation accuracy is 0.9338\n",
      "after 600 training steps, the loss is 0.349948, the validation accuracy is 0.9396\n",
      "after 700 training steps, the loss is 0.422205, the validation accuracy is 0.942\n",
      "after 800 training steps, the loss is 0.320914, the validation accuracy is 0.9372\n",
      "after 900 training steps, the loss is 0.358415, the validation accuracy is 0.941\n",
      "after 1000 training steps, the loss is 0.413265, the validation accuracy is 0.9448\n",
      "after 1100 training steps, the loss is 0.449668, the validation accuracy is 0.9472\n",
      "after 1200 training steps, the loss is 0.293754, the validation accuracy is 0.9514\n",
      "after 1300 training steps, the loss is 0.481817, the validation accuracy is 0.94\n",
      "after 1400 training steps, the loss is 0.303566, the validation accuracy is 0.9472\n",
      "after 1500 training steps, the loss is 0.482139, the validation accuracy is 0.9526\n",
      "after 1600 training steps, the loss is 0.277515, the validation accuracy is 0.9518\n",
      "after 1700 training steps, the loss is 0.24946, the validation accuracy is 0.9488\n",
      "after 1800 training steps, the loss is 0.33371, the validation accuracy is 0.9448\n",
      "after 1900 training steps, the loss is 0.306883, the validation accuracy is 0.9482\n",
      "after 2000 training steps, the loss is 0.269579, the validation accuracy is 0.9546\n",
      "after 2100 training steps, the loss is 0.187644, the validation accuracy is 0.955\n",
      "after 2200 training steps, the loss is 0.16594, the validation accuracy is 0.9534\n",
      "after 2300 training steps, the loss is 0.186246, the validation accuracy is 0.9564\n",
      "after 2400 training steps, the loss is 0.289639, the validation accuracy is 0.9558\n",
      "after 2500 training steps, the loss is 0.303509, the validation accuracy is 0.9542\n",
      "after 2600 training steps, the loss is 0.18772, the validation accuracy is 0.958\n",
      "after 2700 training steps, the loss is 0.185148, the validation accuracy is 0.954\n",
      "after 2800 training steps, the loss is 0.27884, the validation accuracy is 0.9552\n",
      "after 2900 training steps, the loss is 0.120014, the validation accuracy is 0.9584\n",
      "after 3000 training steps, the loss is 0.200626, the validation accuracy is 0.9552\n",
      "after 3100 training steps, the loss is 0.259717, the validation accuracy is 0.9628\n",
      "after 3200 training steps, the loss is 0.316161, the validation accuracy is 0.961\n",
      "after 3300 training steps, the loss is 0.223642, the validation accuracy is 0.96\n",
      "after 3400 training steps, the loss is 0.257076, the validation accuracy is 0.9536\n",
      "after 3500 training steps, the loss is 0.320117, the validation accuracy is 0.9592\n",
      "after 3600 training steps, the loss is 0.216789, the validation accuracy is 0.9606\n",
      "after 3700 training steps, the loss is 0.112337, the validation accuracy is 0.9608\n",
      "after 3800 training steps, the loss is 0.152865, the validation accuracy is 0.9584\n",
      "after 3900 training steps, the loss is 0.150703, the validation accuracy is 0.9578\n",
      "after 4000 training steps, the loss is 0.294723, the validation accuracy is 0.9614\n",
      "after 4100 training steps, the loss is 0.204765, the validation accuracy is 0.9608\n",
      "after 4200 training steps, the loss is 0.262108, the validation accuracy is 0.9564\n",
      "after 4300 training steps, the loss is 0.150041, the validation accuracy is 0.9612\n",
      "after 4400 training steps, the loss is 0.119285, the validation accuracy is 0.9654\n",
      "after 4500 training steps, the loss is 0.137916, the validation accuracy is 0.9642\n",
      "after 4600 training steps, the loss is 0.223498, the validation accuracy is 0.9618\n",
      "after 4700 training steps, the loss is 0.350933, the validation accuracy is 0.9638\n",
      "after 4800 training steps, the loss is 0.117811, the validation accuracy is 0.9642\n",
      "after 4900 training steps, the loss is 0.209839, the validation accuracy is 0.965\n",
      "after 5000 training steps, the loss is 0.192564, the validation accuracy is 0.9568\n",
      "after 5100 training steps, the loss is 0.141129, the validation accuracy is 0.962\n",
      "after 5200 training steps, the loss is 0.374193, the validation accuracy is 0.9634\n",
      "after 5300 training steps, the loss is 0.293517, the validation accuracy is 0.9598\n",
      "after 5400 training steps, the loss is 0.24572, the validation accuracy is 0.9624\n",
      "after 5500 training steps, the loss is 0.203326, the validation accuracy is 0.965\n",
      "after 5600 training steps, the loss is 0.215456, the validation accuracy is 0.9624\n",
      "after 5700 training steps, the loss is 0.325453, the validation accuracy is 0.9624\n",
      "after 5800 training steps, the loss is 0.183345, the validation accuracy is 0.9636\n",
      "after 5900 training steps, the loss is 0.20643, the validation accuracy is 0.9642\n",
      "after 6000 training steps, the loss is 0.24352, the validation accuracy is 0.9646\n",
      "after 6100 training steps, the loss is 0.191303, the validation accuracy is 0.9658\n",
      "after 6200 training steps, the loss is 0.234826, the validation accuracy is 0.966\n",
      "after 6300 training steps, the loss is 0.160802, the validation accuracy is 0.9644\n",
      "after 6400 training steps, the loss is 0.208266, the validation accuracy is 0.9628\n",
      "after 6500 training steps, the loss is 0.410182, the validation accuracy is 0.9622\n",
      "after 6600 training steps, the loss is 0.206803, the validation accuracy is 0.9656\n",
      "after 6700 training steps, the loss is 0.208115, the validation accuracy is 0.9646\n",
      "after 6800 training steps, the loss is 0.399218, the validation accuracy is 0.963\n",
      "after 6900 training steps, the loss is 0.103261, the validation accuracy is 0.967\n",
      "after 7000 training steps, the loss is 0.193818, the validation accuracy is 0.966\n",
      "after 7100 training steps, the loss is 0.142133, the validation accuracy is 0.9618\n",
      "after 7200 training steps, the loss is 0.232747, the validation accuracy is 0.9658\n",
      "after 7300 training steps, the loss is 0.229204, the validation accuracy is 0.9668\n",
      "after 7400 training steps, the loss is 0.354456, the validation accuracy is 0.9678\n",
      "after 7500 training steps, the loss is 0.132473, the validation accuracy is 0.9648\n",
      "after 7600 training steps, the loss is 0.141932, the validation accuracy is 0.9652\n",
      "after 7700 training steps, the loss is 0.121495, the validation accuracy is 0.9662\n",
      "after 7800 training steps, the loss is 0.234157, the validation accuracy is 0.9642\n",
      "after 7900 training steps, the loss is 0.131066, the validation accuracy is 0.9672\n",
      "after 8000 training steps, the loss is 0.128139, the validation accuracy is 0.967\n",
      "after 8100 training steps, the loss is 0.398435, the validation accuracy is 0.9646\n",
      "after 8200 training steps, the loss is 0.175811, the validation accuracy is 0.9644\n",
      "after 8300 training steps, the loss is 0.267571, the validation accuracy is 0.9628\n",
      "after 8400 training steps, the loss is 0.280483, the validation accuracy is 0.968\n",
      "after 8500 training steps, the loss is 0.175008, the validation accuracy is 0.9696\n",
      "after 8600 training steps, the loss is 0.218805, the validation accuracy is 0.9666\n",
      "after 8700 training steps, the loss is 0.293018, the validation accuracy is 0.9658\n",
      "after 8800 training steps, the loss is 0.126124, the validation accuracy is 0.9682\n",
      "after 8900 training steps, the loss is 0.20903, the validation accuracy is 0.9672\n",
      "after 9000 training steps, the loss is 0.199317, the validation accuracy is 0.9664\n",
      "after 9100 training steps, the loss is 0.292558, the validation accuracy is 0.9712\n",
      "after 9200 training steps, the loss is 0.15926, the validation accuracy is 0.965\n",
      "after 9300 training steps, the loss is 0.154485, the validation accuracy is 0.9676\n",
      "after 9400 training steps, the loss is 0.120698, the validation accuracy is 0.9702\n",
      "after 9500 training steps, the loss is 0.334414, the validation accuracy is 0.9582\n",
      "after 9600 training steps, the loss is 0.264238, the validation accuracy is 0.9592\n",
      "after 9700 training steps, the loss is 0.259789, the validation accuracy is 0.9698\n",
      "after 9800 training steps, the loss is 0.150991, the validation accuracy is 0.9688\n",
      "after 9900 training steps, the loss is 0.155746, the validation accuracy is 0.9672\n",
      "after 10000 training steps, the loss is 0.095146, the validation accuracy is 0.969\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10100 training steps, the loss is 0.147064, the validation accuracy is 0.9668\n",
      "after 10200 training steps, the loss is 0.0904064, the validation accuracy is 0.9664\n",
      "after 10300 training steps, the loss is 0.127883, the validation accuracy is 0.968\n",
      "after 10400 training steps, the loss is 0.10177, the validation accuracy is 0.9704\n",
      "after 10500 training steps, the loss is 0.171924, the validation accuracy is 0.97\n",
      "after 10600 training steps, the loss is 0.142643, the validation accuracy is 0.9692\n",
      "after 10700 training steps, the loss is 0.115668, the validation accuracy is 0.9688\n",
      "after 10800 training steps, the loss is 0.134925, the validation accuracy is 0.9704\n",
      "after 10900 training steps, the loss is 0.219021, the validation accuracy is 0.9674\n",
      "after 11000 training steps, the loss is 0.125084, the validation accuracy is 0.9698\n",
      "after 11100 training steps, the loss is 0.121816, the validation accuracy is 0.9676\n",
      "after 11200 training steps, the loss is 0.149938, the validation accuracy is 0.97\n",
      "after 11300 training steps, the loss is 0.118165, the validation accuracy is 0.9676\n",
      "after 11400 training steps, the loss is 0.232455, the validation accuracy is 0.9704\n",
      "after 11500 training steps, the loss is 0.113825, the validation accuracy is 0.9728\n",
      "after 11600 training steps, the loss is 0.130551, the validation accuracy is 0.9692\n",
      "after 11700 training steps, the loss is 0.196291, the validation accuracy is 0.9682\n",
      "after 11800 training steps, the loss is 0.285606, the validation accuracy is 0.97\n",
      "after 11900 training steps, the loss is 0.155032, the validation accuracy is 0.9684\n",
      "after 12000 training steps, the loss is 0.312952, the validation accuracy is 0.9672\n",
      "after 12100 training steps, the loss is 0.158721, the validation accuracy is 0.9702\n",
      "after 12200 training steps, the loss is 0.110351, the validation accuracy is 0.9676\n",
      "after 12300 training steps, the loss is 0.231743, the validation accuracy is 0.9706\n",
      "after 12400 training steps, the loss is 0.113267, the validation accuracy is 0.9698\n",
      "after 12500 training steps, the loss is 0.203192, the validation accuracy is 0.9688\n",
      "after 12600 training steps, the loss is 0.147399, the validation accuracy is 0.9706\n",
      "after 12700 training steps, the loss is 0.177469, the validation accuracy is 0.9688\n",
      "after 12800 training steps, the loss is 0.106744, the validation accuracy is 0.967\n",
      "after 12900 training steps, the loss is 0.0985759, the validation accuracy is 0.9684\n",
      "after 13000 training steps, the loss is 0.187052, the validation accuracy is 0.9712\n",
      "after 13100 training steps, the loss is 0.132073, the validation accuracy is 0.9688\n",
      "after 13200 training steps, the loss is 0.290316, the validation accuracy is 0.969\n",
      "after 13300 training steps, the loss is 0.173805, the validation accuracy is 0.9728\n",
      "after 13400 training steps, the loss is 0.126814, the validation accuracy is 0.9682\n",
      "after 13500 training steps, the loss is 0.300508, the validation accuracy is 0.9718\n",
      "after 13600 training steps, the loss is 0.0767254, the validation accuracy is 0.9676\n",
      "after 13700 training steps, the loss is 0.123127, the validation accuracy is 0.9706\n",
      "after 13800 training steps, the loss is 0.155935, the validation accuracy is 0.9706\n",
      "after 13900 training steps, the loss is 0.296941, the validation accuracy is 0.9696\n",
      "after 14000 training steps, the loss is 0.105602, the validation accuracy is 0.968\n",
      "after 14100 training steps, the loss is 0.121633, the validation accuracy is 0.9694\n",
      "after 14200 training steps, the loss is 0.213202, the validation accuracy is 0.9712\n",
      "after 14300 training steps, the loss is 0.0779245, the validation accuracy is 0.973\n",
      "after 14400 training steps, the loss is 0.102555, the validation accuracy is 0.9748\n",
      "after 14500 training steps, the loss is 0.1959, the validation accuracy is 0.9704\n",
      "after 14600 training steps, the loss is 0.0892212, the validation accuracy is 0.9692\n",
      "after 14700 training steps, the loss is 0.250411, the validation accuracy is 0.9638\n",
      "after 14800 training steps, the loss is 0.104476, the validation accuracy is 0.9744\n",
      "after 14900 training steps, the loss is 0.17076, the validation accuracy is 0.9716\n",
      "after 15000 training steps, the loss is 0.232437, the validation accuracy is 0.9686\n",
      "after 15100 training steps, the loss is 0.129289, the validation accuracy is 0.972\n",
      "after 15200 training steps, the loss is 0.158663, the validation accuracy is 0.9688\n",
      "after 15300 training steps, the loss is 0.158148, the validation accuracy is 0.97\n",
      "after 15400 training steps, the loss is 0.147942, the validation accuracy is 0.9718\n",
      "after 15500 training steps, the loss is 0.291822, the validation accuracy is 0.9646\n",
      "after 15600 training steps, the loss is 0.0877916, the validation accuracy is 0.9696\n",
      "after 15700 training steps, the loss is 0.0837406, the validation accuracy is 0.9722\n",
      "after 15800 training steps, the loss is 0.100241, the validation accuracy is 0.973\n",
      "after 15900 training steps, the loss is 0.183714, the validation accuracy is 0.9724\n",
      "after 16000 training steps, the loss is 0.0767367, the validation accuracy is 0.971\n",
      "after 16100 training steps, the loss is 0.106943, the validation accuracy is 0.9716\n",
      "after 16200 training steps, the loss is 0.318993, the validation accuracy is 0.969\n",
      "after 16300 training steps, the loss is 0.244741, the validation accuracy is 0.9702\n",
      "after 16400 training steps, the loss is 0.215675, the validation accuracy is 0.972\n",
      "after 16500 training steps, the loss is 0.133221, the validation accuracy is 0.9744\n",
      "after 16600 training steps, the loss is 0.0743613, the validation accuracy is 0.972\n",
      "after 16700 training steps, the loss is 0.111173, the validation accuracy is 0.9714\n",
      "after 16800 training steps, the loss is 0.124032, the validation accuracy is 0.9722\n",
      "after 16900 training steps, the loss is 0.155983, the validation accuracy is 0.9714\n",
      "after 17000 training steps, the loss is 0.0731644, the validation accuracy is 0.9714\n",
      "after 17100 training steps, the loss is 0.0934987, the validation accuracy is 0.9712\n",
      "after 17200 training steps, the loss is 0.207181, the validation accuracy is 0.972\n",
      "after 17300 training steps, the loss is 0.161456, the validation accuracy is 0.9736\n",
      "after 17400 training steps, the loss is 0.0869394, the validation accuracy is 0.9728\n",
      "after 17500 training steps, the loss is 0.120995, the validation accuracy is 0.9688\n",
      "after 17600 training steps, the loss is 0.200351, the validation accuracy is 0.9732\n",
      "after 17700 training steps, the loss is 0.103466, the validation accuracy is 0.973\n",
      "after 17800 training steps, the loss is 0.144634, the validation accuracy is 0.9726\n",
      "after 17900 training steps, the loss is 0.252267, the validation accuracy is 0.972\n",
      "after 18000 training steps, the loss is 0.176282, the validation accuracy is 0.9708\n",
      "after 18100 training steps, the loss is 0.149538, the validation accuracy is 0.9722\n",
      "after 18200 training steps, the loss is 0.129395, the validation accuracy is 0.971\n",
      "after 18300 training steps, the loss is 0.132328, the validation accuracy is 0.9722\n",
      "after 18400 training steps, the loss is 0.159071, the validation accuracy is 0.9724\n",
      "after 18500 training steps, the loss is 0.14601, the validation accuracy is 0.974\n",
      "after 18600 training steps, the loss is 0.124307, the validation accuracy is 0.9724\n",
      "after 18700 training steps, the loss is 0.166134, the validation accuracy is 0.9736\n",
      "after 18800 training steps, the loss is 0.105798, the validation accuracy is 0.9738\n",
      "after 18900 training steps, the loss is 0.169311, the validation accuracy is 0.9726\n",
      "after 19000 training steps, the loss is 0.158653, the validation accuracy is 0.9736\n",
      "after 19100 training steps, the loss is 0.155127, the validation accuracy is 0.9748\n",
      "after 19200 training steps, the loss is 0.207291, the validation accuracy is 0.9718\n",
      "after 19300 training steps, the loss is 0.224283, the validation accuracy is 0.971\n",
      "after 19400 training steps, the loss is 0.145579, the validation accuracy is 0.9724\n",
      "after 19500 training steps, the loss is 0.0929979, the validation accuracy is 0.9702\n",
      "after 19600 training steps, the loss is 0.104316, the validation accuracy is 0.9748\n",
      "after 19700 training steps, the loss is 0.108775, the validation accuracy is 0.9738\n",
      "after 19800 training steps, the loss is 0.128911, the validation accuracy is 0.9702\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19900 training steps, the loss is 0.115615, the validation accuracy is 0.972\n",
      "after 20000 training steps, the loss is 0.0785036, the validation accuracy is 0.9758\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.9782909\n",
      "the test accuarcy is: 0.9717\n"
     ]
    }
   ],
   "source": [
    "# l2正则\n",
    "train(batch_size=32, training_step=20000, lr=0.1, hidden1=1000, lambda_flag='l2', lambda_value=0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "l2正则效果比l1要好，通过l2正则将模型的训练集上的准确率和测试集上的准确率差距减小了，都在0.97以上。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 改变权重和偏置项初始化方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.704204, the validation accuracy is 0.7296\n",
      "after 200 training steps, the loss is 0.584972, the validation accuracy is 0.806\n",
      "after 300 training steps, the loss is 0.412275, the validation accuracy is 0.8738\n",
      "after 400 training steps, the loss is 0.890842, the validation accuracy is 0.8438\n",
      "after 500 training steps, the loss is 0.134283, the validation accuracy is 0.9246\n",
      "after 600 training steps, the loss is 0.0561884, the validation accuracy is 0.9236\n",
      "after 700 training steps, the loss is 0.207537, the validation accuracy is 0.9274\n",
      "after 800 training steps, the loss is 0.138588, the validation accuracy is 0.9272\n",
      "after 900 training steps, the loss is 0.38375, the validation accuracy is 0.9354\n",
      "after 1000 training steps, the loss is 0.420443, the validation accuracy is 0.9172\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.9137091\n",
      "the test accuarcy is: 0.9165\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=1000, lr=0.1, hidden1=100, hidden2=100, \n",
    "      hidden3=100, hidden4=100, hidden5=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 1.13898, the validation accuracy is 0.581\n",
      "after 200 training steps, the loss is 0.757329, the validation accuracy is 0.7478\n",
      "after 300 training steps, the loss is 0.320514, the validation accuracy is 0.8424\n",
      "after 400 training steps, the loss is 0.297898, the validation accuracy is 0.886\n",
      "after 500 training steps, the loss is 0.486076, the validation accuracy is 0.8826\n",
      "after 600 training steps, the loss is 0.470115, the validation accuracy is 0.873\n",
      "after 700 training steps, the loss is 0.0784942, the validation accuracy is 0.9282\n",
      "after 800 training steps, the loss is 0.43384, the validation accuracy is 0.925\n",
      "after 900 training steps, the loss is 0.298118, the validation accuracy is 0.9064\n",
      "after 1000 training steps, the loss is 0.0856277, the validation accuracy is 0.935\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.92716366\n",
      "the test accuarcy is: 0.9235\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=1000, lr=0.1, initial_way=Xavier, hidden1=100, hidden2=100,\n",
    "      hidden3=100, hidden4=100,hidden5=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 1.08336, the validation accuracy is 0.5836\n",
      "after 200 training steps, the loss is 0.525242, the validation accuracy is 0.8304\n",
      "after 300 training steps, the loss is 0.179845, the validation accuracy is 0.8638\n",
      "after 400 training steps, the loss is 0.549597, the validation accuracy is 0.8892\n",
      "after 500 training steps, the loss is 0.634014, the validation accuracy is 0.8868\n",
      "after 600 training steps, the loss is 0.12086, the validation accuracy is 0.916\n",
      "after 700 training steps, the loss is 0.340264, the validation accuracy is 0.919\n",
      "after 800 training steps, the loss is 0.387933, the validation accuracy is 0.9146\n",
      "after 900 training steps, the loss is 0.196221, the validation accuracy is 0.9256\n",
      "after 1000 training steps, the loss is 0.0839976, the validation accuracy is 0.9426\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.9365818\n",
      "the test accuarcy is: 0.9358\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=1000, lr=0.1, initial_way=MSRA, hidden1=100, hidden2=100, \n",
    "      hidden3=100, hidden4=100, hidden5=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "随着神经网络层数增加，高斯分布初始化的参数容易出现过大或过小的问题。在其它参数一致仅迭代1000次情况下，Xavier和MSRA初始化方式明显比普通高斯分布初始化方式效果好，对于relu激活函数MSRA初始化方式效果最好。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 综合上面各个模型优化角度，接下来我们把模型变得复杂一些，增加迭代次数训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 2.03621, the validation accuracy is 0.3424\n",
      "after 200 training steps, the loss is 1.58934, the validation accuracy is 0.4306\n",
      "after 300 training steps, the loss is 1.17438, the validation accuracy is 0.7268\n",
      "after 400 training steps, the loss is 1.08909, the validation accuracy is 0.7288\n",
      "after 500 training steps, the loss is 0.997169, the validation accuracy is 0.7898\n",
      "after 600 training steps, the loss is 0.880697, the validation accuracy is 0.8184\n",
      "after 700 training steps, the loss is 0.601813, the validation accuracy is 0.7922\n",
      "after 800 training steps, the loss is 0.81817, the validation accuracy is 0.8114\n",
      "after 900 training steps, the loss is 0.812748, the validation accuracy is 0.8366\n",
      "after 1000 training steps, the loss is 0.629533, the validation accuracy is 0.798\n",
      "after 1100 training steps, the loss is 0.815184, the validation accuracy is 0.8676\n",
      "after 1200 training steps, the loss is 0.489692, the validation accuracy is 0.862\n",
      "after 1300 training steps, the loss is 0.602924, the validation accuracy is 0.8652\n",
      "after 1400 training steps, the loss is 0.627552, the validation accuracy is 0.8826\n",
      "after 1500 training steps, the loss is 0.602489, the validation accuracy is 0.8664\n",
      "after 1600 training steps, the loss is 0.624577, the validation accuracy is 0.8692\n",
      "after 1700 training steps, the loss is 0.642596, the validation accuracy is 0.8772\n",
      "after 1800 training steps, the loss is 0.625199, the validation accuracy is 0.8834\n",
      "after 1900 training steps, the loss is 0.47261, the validation accuracy is 0.8896\n",
      "after 2000 training steps, the loss is 0.363217, the validation accuracy is 0.8786\n",
      "after 2100 training steps, the loss is 0.501104, the validation accuracy is 0.8912\n",
      "after 2200 training steps, the loss is 0.639629, the validation accuracy is 0.89\n",
      "after 2300 training steps, the loss is 0.612983, the validation accuracy is 0.8948\n",
      "after 2400 training steps, the loss is 0.355927, the validation accuracy is 0.8932\n",
      "after 2500 training steps, the loss is 0.509735, the validation accuracy is 0.8908\n",
      "after 2600 training steps, the loss is 0.607301, the validation accuracy is 0.8972\n",
      "after 2700 training steps, the loss is 0.517207, the validation accuracy is 0.896\n",
      "after 2800 training steps, the loss is 0.629813, the validation accuracy is 0.8996\n",
      "after 2900 training steps, the loss is 0.939756, the validation accuracy is 0.8944\n",
      "after 3000 training steps, the loss is 0.46954, the validation accuracy is 0.892\n",
      "after 3100 training steps, the loss is 0.444222, the validation accuracy is 0.9014\n",
      "after 3200 training steps, the loss is 0.428722, the validation accuracy is 0.903\n",
      "after 3300 training steps, the loss is 0.40901, the validation accuracy is 0.9034\n",
      "after 3400 training steps, the loss is 0.598702, the validation accuracy is 0.8966\n",
      "after 3500 training steps, the loss is 0.448483, the validation accuracy is 0.9042\n",
      "after 3600 training steps, the loss is 0.327102, the validation accuracy is 0.9054\n",
      "after 3700 training steps, the loss is 0.617935, the validation accuracy is 0.9028\n",
      "after 3800 training steps, the loss is 0.35997, the validation accuracy is 0.9122\n",
      "after 3900 training steps, the loss is 0.49866, the validation accuracy is 0.9054\n",
      "after 4000 training steps, the loss is 0.61697, the validation accuracy is 0.9054\n",
      "after 4100 training steps, the loss is 0.382196, the validation accuracy is 0.9026\n",
      "after 4200 training steps, the loss is 0.477025, the validation accuracy is 0.9122\n",
      "after 4300 training steps, the loss is 0.282733, the validation accuracy is 0.907\n",
      "after 4400 training steps, the loss is 0.314707, the validation accuracy is 0.91\n",
      "after 4500 training steps, the loss is 0.338838, the validation accuracy is 0.9076\n",
      "after 4600 training steps, the loss is 0.354464, the validation accuracy is 0.9022\n",
      "after 4700 training steps, the loss is 0.35959, the validation accuracy is 0.9084\n",
      "after 4800 training steps, the loss is 0.410387, the validation accuracy is 0.91\n",
      "after 4900 training steps, the loss is 0.554604, the validation accuracy is 0.9124\n",
      "after 5000 training steps, the loss is 0.48037, the validation accuracy is 0.9082\n",
      "after 5100 training steps, the loss is 0.251842, the validation accuracy is 0.9136\n",
      "after 5200 training steps, the loss is 0.373611, the validation accuracy is 0.908\n",
      "after 5300 training steps, the loss is 0.259442, the validation accuracy is 0.917\n",
      "after 5400 training steps, the loss is 0.366098, the validation accuracy is 0.9158\n",
      "after 5500 training steps, the loss is 0.268471, the validation accuracy is 0.9142\n",
      "after 5600 training steps, the loss is 0.551063, the validation accuracy is 0.9166\n",
      "after 5700 training steps, the loss is 0.380455, the validation accuracy is 0.9186\n",
      "after 5800 training steps, the loss is 0.299355, the validation accuracy is 0.9196\n",
      "after 5900 training steps, the loss is 0.347925, the validation accuracy is 0.9166\n",
      "after 6000 training steps, the loss is 0.63508, the validation accuracy is 0.916\n",
      "after 6100 training steps, the loss is 0.283903, the validation accuracy is 0.916\n",
      "after 6200 training steps, the loss is 0.343526, the validation accuracy is 0.9242\n",
      "after 6300 training steps, the loss is 0.47569, the validation accuracy is 0.9186\n",
      "after 6400 training steps, the loss is 0.386205, the validation accuracy is 0.9196\n",
      "after 6500 training steps, the loss is 0.429822, the validation accuracy is 0.9202\n",
      "after 6600 training steps, the loss is 0.46508, the validation accuracy is 0.9152\n",
      "after 6700 training steps, the loss is 0.391833, the validation accuracy is 0.9196\n",
      "after 6800 training steps, the loss is 0.242275, the validation accuracy is 0.9212\n",
      "after 6900 training steps, the loss is 0.276142, the validation accuracy is 0.9262\n",
      "after 7000 training steps, the loss is 0.400995, the validation accuracy is 0.9102\n",
      "after 7100 training steps, the loss is 0.231519, the validation accuracy is 0.9194\n",
      "after 7200 training steps, the loss is 0.243815, the validation accuracy is 0.9228\n",
      "after 7300 training steps, the loss is 0.798419, the validation accuracy is 0.918\n",
      "after 7400 training steps, the loss is 0.236848, the validation accuracy is 0.918\n",
      "after 7500 training steps, the loss is 0.334432, the validation accuracy is 0.9162\n",
      "after 7600 training steps, the loss is 0.387295, the validation accuracy is 0.9258\n",
      "after 7700 training steps, the loss is 0.296025, the validation accuracy is 0.9176\n",
      "after 7800 training steps, the loss is 0.377403, the validation accuracy is 0.919\n",
      "after 7900 training steps, the loss is 0.429507, the validation accuracy is 0.9216\n",
      "after 8000 training steps, the loss is 0.600746, the validation accuracy is 0.9228\n",
      "after 8100 training steps, the loss is 0.376344, the validation accuracy is 0.9284\n",
      "after 8200 training steps, the loss is 0.378466, the validation accuracy is 0.9226\n",
      "after 8300 training steps, the loss is 0.449674, the validation accuracy is 0.9276\n",
      "after 8400 training steps, the loss is 0.269233, the validation accuracy is 0.9286\n",
      "after 8500 training steps, the loss is 0.438013, the validation accuracy is 0.9306\n",
      "after 8600 training steps, the loss is 0.779901, the validation accuracy is 0.92\n",
      "after 8700 training steps, the loss is 0.174243, the validation accuracy is 0.926\n",
      "after 8800 training steps, the loss is 0.388647, the validation accuracy is 0.9164\n",
      "after 8900 training steps, the loss is 0.360376, the validation accuracy is 0.923\n",
      "after 9000 training steps, the loss is 0.211766, the validation accuracy is 0.9272\n",
      "after 9100 training steps, the loss is 0.295199, the validation accuracy is 0.9252\n",
      "after 9200 training steps, the loss is 0.375846, the validation accuracy is 0.9316\n",
      "after 9300 training steps, the loss is 0.290265, the validation accuracy is 0.9224\n",
      "after 9400 training steps, the loss is 0.495023, the validation accuracy is 0.93\n",
      "after 9500 training steps, the loss is 0.431211, the validation accuracy is 0.9276\n",
      "after 9600 training steps, the loss is 0.254286, the validation accuracy is 0.9172\n",
      "after 9700 training steps, the loss is 0.304805, the validation accuracy is 0.9242\n",
      "after 9800 training steps, the loss is 0.447911, the validation accuracy is 0.9276\n",
      "after 9900 training steps, the loss is 0.350767, the validation accuracy is 0.9274\n",
      "after 10000 training steps, the loss is 0.419952, the validation accuracy is 0.93\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10100 training steps, the loss is 0.333244, the validation accuracy is 0.9316\n",
      "after 10200 training steps, the loss is 0.25968, the validation accuracy is 0.9272\n",
      "after 10300 training steps, the loss is 0.42136, the validation accuracy is 0.9266\n",
      "after 10400 training steps, the loss is 0.18591, the validation accuracy is 0.9282\n",
      "after 10500 training steps, the loss is 0.250165, the validation accuracy is 0.9276\n",
      "after 10600 training steps, the loss is 0.532484, the validation accuracy is 0.924\n",
      "after 10700 training steps, the loss is 0.294181, the validation accuracy is 0.931\n",
      "after 10800 training steps, the loss is 0.444149, the validation accuracy is 0.9296\n",
      "after 10900 training steps, the loss is 0.512956, the validation accuracy is 0.9334\n",
      "after 11000 training steps, the loss is 0.301724, the validation accuracy is 0.9334\n",
      "after 11100 training steps, the loss is 0.335779, the validation accuracy is 0.928\n",
      "after 11200 training steps, the loss is 0.22628, the validation accuracy is 0.9336\n",
      "after 11300 training steps, the loss is 0.222881, the validation accuracy is 0.9358\n",
      "after 11400 training steps, the loss is 0.439612, the validation accuracy is 0.9304\n",
      "after 11500 training steps, the loss is 0.540991, the validation accuracy is 0.9306\n",
      "after 11600 training steps, the loss is 0.332744, the validation accuracy is 0.9348\n",
      "after 11700 training steps, the loss is 0.292873, the validation accuracy is 0.935\n",
      "after 11800 training steps, the loss is 0.386055, the validation accuracy is 0.9302\n",
      "after 11900 training steps, the loss is 0.297169, the validation accuracy is 0.9276\n",
      "after 12000 training steps, the loss is 0.37775, the validation accuracy is 0.9308\n",
      "after 12100 training steps, the loss is 0.134688, the validation accuracy is 0.9362\n",
      "after 12200 training steps, the loss is 0.414782, the validation accuracy is 0.9206\n",
      "after 12300 training steps, the loss is 0.203797, the validation accuracy is 0.9358\n",
      "after 12400 training steps, the loss is 0.210836, the validation accuracy is 0.9346\n",
      "after 12500 training steps, the loss is 0.152959, the validation accuracy is 0.9336\n",
      "after 12600 training steps, the loss is 0.323758, the validation accuracy is 0.9366\n",
      "after 12700 training steps, the loss is 0.149871, the validation accuracy is 0.9368\n",
      "after 12800 training steps, the loss is 0.324169, the validation accuracy is 0.9282\n",
      "after 12900 training steps, the loss is 0.341039, the validation accuracy is 0.935\n",
      "after 13000 training steps, the loss is 0.174612, the validation accuracy is 0.935\n",
      "after 13100 training steps, the loss is 0.416085, the validation accuracy is 0.9352\n",
      "after 13200 training steps, the loss is 0.176368, the validation accuracy is 0.9304\n",
      "after 13300 training steps, the loss is 0.35811, the validation accuracy is 0.9396\n",
      "after 13400 training steps, the loss is 0.358769, the validation accuracy is 0.9316\n",
      "after 13500 training steps, the loss is 0.215878, the validation accuracy is 0.9348\n",
      "after 13600 training steps, the loss is 0.343656, the validation accuracy is 0.937\n",
      "after 13700 training steps, the loss is 0.358745, the validation accuracy is 0.9364\n",
      "after 13800 training steps, the loss is 0.441595, the validation accuracy is 0.9382\n",
      "after 13900 training steps, the loss is 0.620427, the validation accuracy is 0.937\n",
      "after 14000 training steps, the loss is 0.26887, the validation accuracy is 0.936\n",
      "after 14100 training steps, the loss is 0.251426, the validation accuracy is 0.936\n",
      "after 14200 training steps, the loss is 0.197187, the validation accuracy is 0.938\n",
      "after 14300 training steps, the loss is 0.200781, the validation accuracy is 0.9388\n",
      "after 14400 training steps, the loss is 0.165756, the validation accuracy is 0.9396\n",
      "after 14500 training steps, the loss is 0.20716, the validation accuracy is 0.9406\n",
      "after 14600 training steps, the loss is 0.211579, the validation accuracy is 0.9366\n",
      "after 14700 training steps, the loss is 0.186976, the validation accuracy is 0.942\n",
      "after 14800 training steps, the loss is 0.147522, the validation accuracy is 0.9382\n",
      "after 14900 training steps, the loss is 0.301579, the validation accuracy is 0.9364\n",
      "after 15000 training steps, the loss is 0.178022, the validation accuracy is 0.9402\n",
      "after 15100 training steps, the loss is 0.476693, the validation accuracy is 0.9362\n",
      "after 15200 training steps, the loss is 0.259066, the validation accuracy is 0.9394\n",
      "after 15300 training steps, the loss is 0.246465, the validation accuracy is 0.9388\n",
      "after 15400 training steps, the loss is 0.34479, the validation accuracy is 0.9324\n",
      "after 15500 training steps, the loss is 0.391211, the validation accuracy is 0.939\n",
      "after 15600 training steps, the loss is 0.339499, the validation accuracy is 0.9388\n",
      "after 15700 training steps, the loss is 0.250995, the validation accuracy is 0.9412\n",
      "after 15800 training steps, the loss is 0.336556, the validation accuracy is 0.9426\n",
      "after 15900 training steps, the loss is 0.192231, the validation accuracy is 0.9432\n",
      "after 16000 training steps, the loss is 0.164617, the validation accuracy is 0.937\n",
      "after 16100 training steps, the loss is 0.405288, the validation accuracy is 0.9388\n",
      "after 16200 training steps, the loss is 0.206613, the validation accuracy is 0.9398\n",
      "after 16300 training steps, the loss is 0.258133, the validation accuracy is 0.9388\n",
      "after 16400 training steps, the loss is 0.585295, the validation accuracy is 0.9406\n",
      "after 16500 training steps, the loss is 0.325626, the validation accuracy is 0.9442\n",
      "after 16600 training steps, the loss is 0.338408, the validation accuracy is 0.9452\n",
      "after 16700 training steps, the loss is 0.265647, the validation accuracy is 0.9378\n",
      "after 16800 training steps, the loss is 0.375942, the validation accuracy is 0.9402\n",
      "after 16900 training steps, the loss is 0.271406, the validation accuracy is 0.9432\n",
      "after 17000 training steps, the loss is 0.192599, the validation accuracy is 0.942\n",
      "after 17100 training steps, the loss is 0.224753, the validation accuracy is 0.9454\n",
      "after 17200 training steps, the loss is 0.384631, the validation accuracy is 0.9406\n",
      "after 17300 training steps, the loss is 0.24556, the validation accuracy is 0.942\n",
      "after 17400 training steps, the loss is 0.304782, the validation accuracy is 0.9386\n",
      "after 17500 training steps, the loss is 0.151601, the validation accuracy is 0.9442\n",
      "after 17600 training steps, the loss is 0.217737, the validation accuracy is 0.9434\n",
      "after 17700 training steps, the loss is 0.491752, the validation accuracy is 0.944\n",
      "after 17800 training steps, the loss is 0.386824, the validation accuracy is 0.9454\n",
      "after 17900 training steps, the loss is 0.276622, the validation accuracy is 0.9356\n",
      "after 18000 training steps, the loss is 0.296411, the validation accuracy is 0.946\n",
      "after 18100 training steps, the loss is 0.288731, the validation accuracy is 0.945\n",
      "after 18200 training steps, the loss is 0.368517, the validation accuracy is 0.9476\n",
      "after 18300 training steps, the loss is 0.263164, the validation accuracy is 0.9484\n",
      "after 18400 training steps, the loss is 0.199585, the validation accuracy is 0.945\n",
      "after 18500 training steps, the loss is 0.210573, the validation accuracy is 0.9424\n",
      "after 18600 training steps, the loss is 0.402826, the validation accuracy is 0.9434\n",
      "after 18700 training steps, the loss is 0.189777, the validation accuracy is 0.9444\n",
      "after 18800 training steps, the loss is 0.403751, the validation accuracy is 0.9464\n",
      "after 18900 training steps, the loss is 0.183576, the validation accuracy is 0.9476\n",
      "after 19000 training steps, the loss is 0.42213, the validation accuracy is 0.947\n",
      "after 19100 training steps, the loss is 0.498139, the validation accuracy is 0.9434\n",
      "after 19200 training steps, the loss is 0.277919, the validation accuracy is 0.9474\n",
      "after 19300 training steps, the loss is 0.154871, the validation accuracy is 0.9422\n",
      "after 19400 training steps, the loss is 0.217365, the validation accuracy is 0.9466\n",
      "after 19500 training steps, the loss is 0.27883, the validation accuracy is 0.945\n",
      "after 19600 training steps, the loss is 0.178016, the validation accuracy is 0.9438\n",
      "after 19700 training steps, the loss is 0.407268, the validation accuracy is 0.9456\n",
      "after 19800 training steps, the loss is 0.297617, the validation accuracy is 0.9426\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19900 training steps, the loss is 0.28433, the validation accuracy is 0.9464\n",
      "after 20000 training steps, the loss is 0.270721, the validation accuracy is 0.937\n",
      "after 20100 training steps, the loss is 0.324518, the validation accuracy is 0.949\n",
      "after 20200 training steps, the loss is 0.107214, the validation accuracy is 0.9458\n",
      "after 20300 training steps, the loss is 0.168703, the validation accuracy is 0.9458\n",
      "after 20400 training steps, the loss is 0.338435, the validation accuracy is 0.9422\n",
      "after 20500 training steps, the loss is 0.202533, the validation accuracy is 0.9404\n",
      "after 20600 training steps, the loss is 0.143797, the validation accuracy is 0.9468\n",
      "after 20700 training steps, the loss is 0.232922, the validation accuracy is 0.9468\n",
      "after 20800 training steps, the loss is 0.243724, the validation accuracy is 0.949\n",
      "after 20900 training steps, the loss is 0.245819, the validation accuracy is 0.9408\n",
      "after 21000 training steps, the loss is 0.230455, the validation accuracy is 0.945\n",
      "after 21100 training steps, the loss is 0.309405, the validation accuracy is 0.9424\n",
      "after 21200 training steps, the loss is 0.192804, the validation accuracy is 0.951\n",
      "after 21300 training steps, the loss is 0.45891, the validation accuracy is 0.9426\n",
      "after 21400 training steps, the loss is 0.135673, the validation accuracy is 0.9502\n",
      "after 21500 training steps, the loss is 0.357102, the validation accuracy is 0.9394\n",
      "after 21600 training steps, the loss is 0.150651, the validation accuracy is 0.945\n",
      "after 21700 training steps, the loss is 0.394788, the validation accuracy is 0.9472\n",
      "after 21800 training steps, the loss is 0.180971, the validation accuracy is 0.9506\n",
      "after 21900 training steps, the loss is 0.292058, the validation accuracy is 0.9496\n",
      "after 22000 training steps, the loss is 0.363395, the validation accuracy is 0.9492\n",
      "after 22100 training steps, the loss is 0.221848, the validation accuracy is 0.9496\n",
      "after 22200 training steps, the loss is 0.421293, the validation accuracy is 0.95\n",
      "after 22300 training steps, the loss is 0.268009, the validation accuracy is 0.9466\n",
      "after 22400 training steps, the loss is 0.223313, the validation accuracy is 0.944\n",
      "after 22500 training steps, the loss is 0.280983, the validation accuracy is 0.952\n",
      "after 22600 training steps, the loss is 0.150027, the validation accuracy is 0.949\n",
      "after 22700 training steps, the loss is 0.393777, the validation accuracy is 0.9478\n",
      "after 22800 training steps, the loss is 0.272797, the validation accuracy is 0.9512\n",
      "after 22900 training steps, the loss is 0.214431, the validation accuracy is 0.9508\n",
      "after 23000 training steps, the loss is 0.221102, the validation accuracy is 0.9506\n",
      "after 23100 training steps, the loss is 0.140772, the validation accuracy is 0.9504\n",
      "after 23200 training steps, the loss is 0.204795, the validation accuracy is 0.9502\n",
      "after 23300 training steps, the loss is 0.150516, the validation accuracy is 0.953\n",
      "after 23400 training steps, the loss is 0.270653, the validation accuracy is 0.952\n",
      "after 23500 training steps, the loss is 0.283656, the validation accuracy is 0.9498\n",
      "after 23600 training steps, the loss is 0.25181, the validation accuracy is 0.9456\n",
      "after 23700 training steps, the loss is 0.252475, the validation accuracy is 0.9508\n",
      "after 23800 training steps, the loss is 0.37613, the validation accuracy is 0.9498\n",
      "after 23900 training steps, the loss is 0.175962, the validation accuracy is 0.9512\n",
      "after 24000 training steps, the loss is 0.177587, the validation accuracy is 0.952\n",
      "after 24100 training steps, the loss is 0.167662, the validation accuracy is 0.9524\n",
      "after 24200 training steps, the loss is 0.319849, the validation accuracy is 0.9458\n",
      "after 24300 training steps, the loss is 0.148432, the validation accuracy is 0.948\n",
      "after 24400 training steps, the loss is 0.3011, the validation accuracy is 0.9498\n",
      "after 24500 training steps, the loss is 0.389976, the validation accuracy is 0.9466\n",
      "after 24600 training steps, the loss is 0.329622, the validation accuracy is 0.9542\n",
      "after 24700 training steps, the loss is 0.181725, the validation accuracy is 0.955\n",
      "after 24800 training steps, the loss is 0.322438, the validation accuracy is 0.9488\n",
      "after 24900 training steps, the loss is 0.143919, the validation accuracy is 0.95\n",
      "after 25000 training steps, the loss is 0.14693, the validation accuracy is 0.9544\n",
      "after 25100 training steps, the loss is 0.270665, the validation accuracy is 0.9508\n",
      "after 25200 training steps, the loss is 0.516491, the validation accuracy is 0.9514\n",
      "after 25300 training steps, the loss is 0.183082, the validation accuracy is 0.952\n",
      "after 25400 training steps, the loss is 0.285791, the validation accuracy is 0.9546\n",
      "after 25500 training steps, the loss is 0.164212, the validation accuracy is 0.95\n",
      "after 25600 training steps, the loss is 0.297432, the validation accuracy is 0.9524\n",
      "after 25700 training steps, the loss is 0.20939, the validation accuracy is 0.9522\n",
      "after 25800 training steps, the loss is 0.384062, the validation accuracy is 0.9484\n",
      "after 25900 training steps, the loss is 0.115809, the validation accuracy is 0.9512\n",
      "after 26000 training steps, the loss is 0.314001, the validation accuracy is 0.9536\n",
      "after 26100 training steps, the loss is 0.139886, the validation accuracy is 0.951\n",
      "after 26200 training steps, the loss is 0.265486, the validation accuracy is 0.9496\n",
      "after 26300 training steps, the loss is 0.385613, the validation accuracy is 0.9546\n",
      "after 26400 training steps, the loss is 0.339454, the validation accuracy is 0.9526\n",
      "after 26500 training steps, the loss is 0.250932, the validation accuracy is 0.9508\n",
      "after 26600 training steps, the loss is 0.303867, the validation accuracy is 0.9548\n",
      "after 26700 training steps, the loss is 0.156747, the validation accuracy is 0.9528\n",
      "after 26800 training steps, the loss is 0.176828, the validation accuracy is 0.952\n",
      "after 26900 training steps, the loss is 0.321658, the validation accuracy is 0.9542\n",
      "after 27000 training steps, the loss is 0.183751, the validation accuracy is 0.949\n",
      "after 27100 training steps, the loss is 0.143247, the validation accuracy is 0.9556\n",
      "after 27200 training steps, the loss is 0.241885, the validation accuracy is 0.9536\n",
      "after 27300 training steps, the loss is 0.285143, the validation accuracy is 0.9544\n",
      "after 27400 training steps, the loss is 0.268957, the validation accuracy is 0.9526\n",
      "after 27500 training steps, the loss is 0.161508, the validation accuracy is 0.9544\n",
      "after 27600 training steps, the loss is 0.177608, the validation accuracy is 0.9536\n",
      "after 27700 training steps, the loss is 0.223187, the validation accuracy is 0.9552\n",
      "after 27800 training steps, the loss is 0.217539, the validation accuracy is 0.955\n",
      "after 27900 training steps, the loss is 0.18585, the validation accuracy is 0.9494\n",
      "after 28000 training steps, the loss is 0.233902, the validation accuracy is 0.9528\n",
      "after 28100 training steps, the loss is 0.374736, the validation accuracy is 0.9546\n",
      "after 28200 training steps, the loss is 0.227307, the validation accuracy is 0.9558\n",
      "after 28300 training steps, the loss is 0.163209, the validation accuracy is 0.9556\n",
      "after 28400 training steps, the loss is 0.141663, the validation accuracy is 0.9538\n",
      "after 28500 training steps, the loss is 0.123138, the validation accuracy is 0.9572\n",
      "after 28600 training steps, the loss is 0.200209, the validation accuracy is 0.9556\n",
      "after 28700 training steps, the loss is 0.247709, the validation accuracy is 0.957\n",
      "after 28800 training steps, the loss is 0.351803, the validation accuracy is 0.9506\n",
      "after 28900 training steps, the loss is 0.365475, the validation accuracy is 0.9584\n",
      "after 29000 training steps, the loss is 0.167856, the validation accuracy is 0.9524\n",
      "after 29100 training steps, the loss is 0.217907, the validation accuracy is 0.9562\n",
      "after 29200 training steps, the loss is 0.287124, the validation accuracy is 0.9564\n",
      "after 29300 training steps, the loss is 0.290873, the validation accuracy is 0.9548\n",
      "after 29400 training steps, the loss is 0.751966, the validation accuracy is 0.9532\n",
      "after 29500 training steps, the loss is 0.232327, the validation accuracy is 0.9568\n",
      "after 29600 training steps, the loss is 0.189151, the validation accuracy is 0.9564\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 29700 training steps, the loss is 0.128401, the validation accuracy is 0.9568\n",
      "after 29800 training steps, the loss is 0.185121, the validation accuracy is 0.9566\n",
      "after 29900 training steps, the loss is 0.121906, the validation accuracy is 0.9554\n",
      "after 30000 training steps, the loss is 0.259415, the validation accuracy is 0.9506\n",
      "after 30100 training steps, the loss is 0.230479, the validation accuracy is 0.9586\n",
      "after 30200 training steps, the loss is 0.213597, the validation accuracy is 0.9518\n",
      "after 30300 training steps, the loss is 0.187463, the validation accuracy is 0.956\n",
      "after 30400 training steps, the loss is 0.13673, the validation accuracy is 0.9588\n",
      "after 30500 training steps, the loss is 0.132087, the validation accuracy is 0.9552\n",
      "after 30600 training steps, the loss is 0.274715, the validation accuracy is 0.9528\n",
      "after 30700 training steps, the loss is 0.182582, the validation accuracy is 0.9572\n",
      "after 30800 training steps, the loss is 0.250438, the validation accuracy is 0.9558\n",
      "after 30900 training steps, the loss is 0.324918, the validation accuracy is 0.9584\n",
      "after 31000 training steps, the loss is 0.162653, the validation accuracy is 0.9542\n",
      "after 31100 training steps, the loss is 0.26, the validation accuracy is 0.9568\n",
      "after 31200 training steps, the loss is 0.121258, the validation accuracy is 0.9586\n",
      "after 31300 training steps, the loss is 0.145336, the validation accuracy is 0.9548\n",
      "after 31400 training steps, the loss is 0.146942, the validation accuracy is 0.957\n",
      "after 31500 training steps, the loss is 0.308724, the validation accuracy is 0.9554\n",
      "after 31600 training steps, the loss is 0.243285, the validation accuracy is 0.9582\n",
      "after 31700 training steps, the loss is 0.143288, the validation accuracy is 0.9578\n",
      "after 31800 training steps, the loss is 0.19257, the validation accuracy is 0.9596\n",
      "after 31900 training steps, the loss is 0.424975, the validation accuracy is 0.955\n",
      "after 32000 training steps, the loss is 0.235494, the validation accuracy is 0.9586\n",
      "after 32100 training steps, the loss is 0.270502, the validation accuracy is 0.958\n",
      "after 32200 training steps, the loss is 0.259143, the validation accuracy is 0.959\n",
      "after 32300 training steps, the loss is 0.311433, the validation accuracy is 0.9564\n",
      "after 32400 training steps, the loss is 0.189478, the validation accuracy is 0.958\n",
      "after 32500 training steps, the loss is 0.16388, the validation accuracy is 0.9568\n",
      "after 32600 training steps, the loss is 0.160458, the validation accuracy is 0.957\n",
      "after 32700 training steps, the loss is 0.239955, the validation accuracy is 0.955\n",
      "after 32800 training steps, the loss is 0.117599, the validation accuracy is 0.9598\n",
      "after 32900 training steps, the loss is 0.127002, the validation accuracy is 0.9566\n",
      "after 33000 training steps, the loss is 0.222958, the validation accuracy is 0.9554\n",
      "after 33100 training steps, the loss is 0.136237, the validation accuracy is 0.9592\n",
      "after 33200 training steps, the loss is 0.203999, the validation accuracy is 0.958\n",
      "after 33300 training steps, the loss is 0.170638, the validation accuracy is 0.958\n",
      "after 33400 training steps, the loss is 0.191446, the validation accuracy is 0.9562\n",
      "after 33500 training steps, the loss is 0.133951, the validation accuracy is 0.957\n",
      "after 33600 training steps, the loss is 0.204123, the validation accuracy is 0.9546\n",
      "after 33700 training steps, the loss is 0.145082, the validation accuracy is 0.9604\n",
      "after 33800 training steps, the loss is 0.15881, the validation accuracy is 0.9542\n",
      "after 33900 training steps, the loss is 0.205941, the validation accuracy is 0.9524\n",
      "after 34000 training steps, the loss is 0.164406, the validation accuracy is 0.9584\n",
      "after 34100 training steps, the loss is 0.20322, the validation accuracy is 0.9578\n",
      "after 34200 training steps, the loss is 0.202705, the validation accuracy is 0.958\n",
      "after 34300 training steps, the loss is 0.144339, the validation accuracy is 0.9598\n",
      "after 34400 training steps, the loss is 0.217679, the validation accuracy is 0.9568\n",
      "after 34500 training steps, the loss is 0.203685, the validation accuracy is 0.958\n",
      "after 34600 training steps, the loss is 0.222513, the validation accuracy is 0.9612\n",
      "after 34700 training steps, the loss is 0.187631, the validation accuracy is 0.9568\n",
      "after 34800 training steps, the loss is 0.156786, the validation accuracy is 0.9598\n",
      "after 34900 training steps, the loss is 0.223644, the validation accuracy is 0.9586\n",
      "after 35000 training steps, the loss is 0.158085, the validation accuracy is 0.961\n",
      "after 35100 training steps, the loss is 0.189016, the validation accuracy is 0.9544\n",
      "after 35200 training steps, the loss is 0.430334, the validation accuracy is 0.9596\n",
      "after 35300 training steps, the loss is 0.306506, the validation accuracy is 0.9576\n",
      "after 35400 training steps, the loss is 0.18298, the validation accuracy is 0.9594\n",
      "after 35500 training steps, the loss is 0.103843, the validation accuracy is 0.9588\n",
      "after 35600 training steps, the loss is 0.111022, the validation accuracy is 0.9594\n",
      "after 35700 training steps, the loss is 0.263419, the validation accuracy is 0.9608\n",
      "after 35800 training steps, the loss is 0.249127, the validation accuracy is 0.9616\n",
      "after 35900 training steps, the loss is 0.145631, the validation accuracy is 0.9582\n",
      "after 36000 training steps, the loss is 0.139118, the validation accuracy is 0.9584\n",
      "after 36100 training steps, the loss is 0.144531, the validation accuracy is 0.9604\n",
      "after 36200 training steps, the loss is 0.312962, the validation accuracy is 0.9628\n",
      "after 36300 training steps, the loss is 0.291741, the validation accuracy is 0.9586\n",
      "after 36400 training steps, the loss is 0.194323, the validation accuracy is 0.9584\n",
      "after 36500 training steps, the loss is 0.135662, the validation accuracy is 0.9618\n",
      "after 36600 training steps, the loss is 0.118775, the validation accuracy is 0.9616\n",
      "after 36700 training steps, the loss is 0.419212, the validation accuracy is 0.9606\n",
      "after 36800 training steps, the loss is 0.282134, the validation accuracy is 0.963\n",
      "after 36900 training steps, the loss is 0.143271, the validation accuracy is 0.9608\n",
      "after 37000 training steps, the loss is 0.220899, the validation accuracy is 0.9612\n",
      "after 37100 training steps, the loss is 0.190742, the validation accuracy is 0.9608\n",
      "after 37200 training steps, the loss is 0.0963383, the validation accuracy is 0.9608\n",
      "after 37300 training steps, the loss is 0.158249, the validation accuracy is 0.9582\n",
      "after 37400 training steps, the loss is 0.127615, the validation accuracy is 0.962\n",
      "after 37500 training steps, the loss is 0.227171, the validation accuracy is 0.9584\n",
      "after 37600 training steps, the loss is 0.384514, the validation accuracy is 0.9596\n",
      "after 37700 training steps, the loss is 0.234767, the validation accuracy is 0.9606\n",
      "after 37800 training steps, the loss is 0.290523, the validation accuracy is 0.9626\n",
      "after 37900 training steps, the loss is 0.178671, the validation accuracy is 0.9608\n",
      "after 38000 training steps, the loss is 0.160555, the validation accuracy is 0.96\n",
      "after 38100 training steps, the loss is 0.0849643, the validation accuracy is 0.96\n",
      "after 38200 training steps, the loss is 0.128714, the validation accuracy is 0.9606\n",
      "after 38300 training steps, the loss is 0.234465, the validation accuracy is 0.953\n",
      "after 38400 training steps, the loss is 0.162734, the validation accuracy is 0.961\n",
      "after 38500 training steps, the loss is 0.265273, the validation accuracy is 0.961\n",
      "after 38600 training steps, the loss is 0.100099, the validation accuracy is 0.9608\n",
      "after 38700 training steps, the loss is 0.120889, the validation accuracy is 0.963\n",
      "after 38800 training steps, the loss is 0.358624, the validation accuracy is 0.9596\n",
      "after 38900 training steps, the loss is 0.381969, the validation accuracy is 0.9632\n",
      "after 39000 training steps, the loss is 0.11806, the validation accuracy is 0.9624\n",
      "after 39100 training steps, the loss is 0.181449, the validation accuracy is 0.9632\n",
      "after 39200 training steps, the loss is 0.171104, the validation accuracy is 0.9602\n",
      "after 39300 training steps, the loss is 0.168496, the validation accuracy is 0.9612\n",
      "after 39400 training steps, the loss is 0.287029, the validation accuracy is 0.9592\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 39500 training steps, the loss is 0.259384, the validation accuracy is 0.96\n",
      "after 39600 training steps, the loss is 0.284048, the validation accuracy is 0.9616\n",
      "after 39700 training steps, the loss is 0.165911, the validation accuracy is 0.9618\n",
      "after 39800 training steps, the loss is 0.146223, the validation accuracy is 0.961\n",
      "after 39900 training steps, the loss is 0.242489, the validation accuracy is 0.9618\n",
      "after 40000 training steps, the loss is 0.25744, the validation accuracy is 0.9604\n",
      "after 40100 training steps, the loss is 0.12893, the validation accuracy is 0.9604\n",
      "after 40200 training steps, the loss is 0.188426, the validation accuracy is 0.9618\n",
      "after 40300 training steps, the loss is 0.195327, the validation accuracy is 0.9622\n",
      "after 40400 training steps, the loss is 0.250291, the validation accuracy is 0.9592\n",
      "after 40500 training steps, the loss is 0.46524, the validation accuracy is 0.9608\n",
      "after 40600 training steps, the loss is 0.328583, the validation accuracy is 0.9626\n",
      "after 40700 training steps, the loss is 0.220472, the validation accuracy is 0.96\n",
      "after 40800 training steps, the loss is 0.218826, the validation accuracy is 0.9612\n",
      "after 40900 training steps, the loss is 0.136411, the validation accuracy is 0.9646\n",
      "after 41000 training steps, the loss is 0.131579, the validation accuracy is 0.9642\n",
      "after 41100 training steps, the loss is 0.120721, the validation accuracy is 0.9618\n",
      "after 41200 training steps, the loss is 0.371647, the validation accuracy is 0.9638\n",
      "after 41300 training steps, the loss is 0.214897, the validation accuracy is 0.9644\n",
      "after 41400 training steps, the loss is 0.121057, the validation accuracy is 0.9614\n",
      "after 41500 training steps, the loss is 0.250837, the validation accuracy is 0.962\n",
      "after 41600 training steps, the loss is 0.357701, the validation accuracy is 0.959\n",
      "after 41700 training steps, the loss is 0.123072, the validation accuracy is 0.9578\n",
      "after 41800 training steps, the loss is 0.14075, the validation accuracy is 0.9646\n",
      "after 41900 training steps, the loss is 0.189414, the validation accuracy is 0.9636\n",
      "after 42000 training steps, the loss is 0.100399, the validation accuracy is 0.9632\n",
      "after 42100 training steps, the loss is 0.115254, the validation accuracy is 0.963\n",
      "after 42200 training steps, the loss is 0.12343, the validation accuracy is 0.9622\n",
      "after 42300 training steps, the loss is 0.171632, the validation accuracy is 0.9632\n",
      "after 42400 training steps, the loss is 0.481533, the validation accuracy is 0.9578\n",
      "after 42500 training steps, the loss is 0.231538, the validation accuracy is 0.962\n",
      "after 42600 training steps, the loss is 0.0750533, the validation accuracy is 0.9614\n",
      "after 42700 training steps, the loss is 0.0961185, the validation accuracy is 0.965\n",
      "after 42800 training steps, the loss is 0.193835, the validation accuracy is 0.963\n",
      "after 42900 training steps, the loss is 0.164197, the validation accuracy is 0.9614\n",
      "after 43000 training steps, the loss is 0.168807, the validation accuracy is 0.963\n",
      "after 43100 training steps, the loss is 0.337821, the validation accuracy is 0.963\n",
      "after 43200 training steps, the loss is 0.267145, the validation accuracy is 0.9614\n",
      "after 43300 training steps, the loss is 0.215808, the validation accuracy is 0.9652\n",
      "after 43400 training steps, the loss is 0.221331, the validation accuracy is 0.965\n",
      "after 43500 training steps, the loss is 0.21151, the validation accuracy is 0.9656\n",
      "after 43600 training steps, the loss is 0.137945, the validation accuracy is 0.964\n",
      "after 43700 training steps, the loss is 0.118244, the validation accuracy is 0.9632\n",
      "after 43800 training steps, the loss is 0.0942187, the validation accuracy is 0.9632\n",
      "after 43900 training steps, the loss is 0.338463, the validation accuracy is 0.9656\n",
      "after 44000 training steps, the loss is 0.296238, the validation accuracy is 0.9624\n",
      "after 44100 training steps, the loss is 0.0904934, the validation accuracy is 0.9636\n",
      "after 44200 training steps, the loss is 0.333139, the validation accuracy is 0.963\n",
      "after 44300 training steps, the loss is 0.10237, the validation accuracy is 0.965\n",
      "after 44400 training steps, the loss is 0.117519, the validation accuracy is 0.9634\n",
      "after 44500 training steps, the loss is 0.104336, the validation accuracy is 0.9598\n",
      "after 44600 training steps, the loss is 0.075299, the validation accuracy is 0.9636\n",
      "after 44700 training steps, the loss is 0.191522, the validation accuracy is 0.9664\n",
      "after 44800 training steps, the loss is 0.156953, the validation accuracy is 0.9652\n",
      "after 44900 training steps, the loss is 0.191773, the validation accuracy is 0.964\n",
      "after 45000 training steps, the loss is 0.244332, the validation accuracy is 0.9636\n",
      "after 45100 training steps, the loss is 0.222903, the validation accuracy is 0.9644\n",
      "after 45200 training steps, the loss is 0.30151, the validation accuracy is 0.96\n",
      "after 45300 training steps, the loss is 0.162609, the validation accuracy is 0.966\n",
      "after 45400 training steps, the loss is 0.224061, the validation accuracy is 0.9628\n",
      "after 45500 training steps, the loss is 0.139882, the validation accuracy is 0.9616\n",
      "after 45600 training steps, the loss is 0.124382, the validation accuracy is 0.9646\n",
      "after 45700 training steps, the loss is 0.179493, the validation accuracy is 0.9626\n",
      "after 45800 training steps, the loss is 0.136758, the validation accuracy is 0.9646\n",
      "after 45900 training steps, the loss is 0.116774, the validation accuracy is 0.963\n",
      "after 46000 training steps, the loss is 0.141949, the validation accuracy is 0.9648\n",
      "after 46100 training steps, the loss is 0.102826, the validation accuracy is 0.9608\n",
      "after 46200 training steps, the loss is 0.0860884, the validation accuracy is 0.964\n",
      "after 46300 training steps, the loss is 0.122189, the validation accuracy is 0.9662\n",
      "after 46400 training steps, the loss is 0.257371, the validation accuracy is 0.9664\n",
      "after 46500 training steps, the loss is 0.225599, the validation accuracy is 0.9654\n",
      "after 46600 training steps, the loss is 0.079873, the validation accuracy is 0.967\n",
      "after 46700 training steps, the loss is 0.0782387, the validation accuracy is 0.9664\n",
      "after 46800 training steps, the loss is 0.235169, the validation accuracy is 0.965\n",
      "after 46900 training steps, the loss is 0.0856958, the validation accuracy is 0.9674\n",
      "after 47000 training steps, the loss is 0.108907, the validation accuracy is 0.9658\n",
      "after 47100 training steps, the loss is 0.291882, the validation accuracy is 0.9672\n",
      "after 47200 training steps, the loss is 0.122123, the validation accuracy is 0.9666\n",
      "after 47300 training steps, the loss is 0.09026, the validation accuracy is 0.9646\n",
      "after 47400 training steps, the loss is 0.252464, the validation accuracy is 0.9652\n",
      "after 47500 training steps, the loss is 0.227886, the validation accuracy is 0.9616\n",
      "after 47600 training steps, the loss is 0.153184, the validation accuracy is 0.9658\n",
      "after 47700 training steps, the loss is 0.252493, the validation accuracy is 0.9648\n",
      "after 47800 training steps, the loss is 0.272598, the validation accuracy is 0.9632\n",
      "after 47900 training steps, the loss is 0.111143, the validation accuracy is 0.9646\n",
      "after 48000 training steps, the loss is 0.137073, the validation accuracy is 0.962\n",
      "after 48100 training steps, the loss is 0.0824542, the validation accuracy is 0.9662\n",
      "after 48200 training steps, the loss is 0.214145, the validation accuracy is 0.9646\n",
      "after 48300 training steps, the loss is 0.267714, the validation accuracy is 0.9642\n",
      "after 48400 training steps, the loss is 0.0998974, the validation accuracy is 0.965\n",
      "after 48500 training steps, the loss is 0.171847, the validation accuracy is 0.9646\n",
      "after 48600 training steps, the loss is 0.207976, the validation accuracy is 0.9662\n",
      "after 48700 training steps, the loss is 0.132429, the validation accuracy is 0.963\n",
      "after 48800 training steps, the loss is 0.237518, the validation accuracy is 0.9632\n",
      "after 48900 training steps, the loss is 0.185328, the validation accuracy is 0.9652\n",
      "after 49000 training steps, the loss is 0.0962045, the validation accuracy is 0.965\n",
      "after 49100 training steps, the loss is 0.359485, the validation accuracy is 0.965\n",
      "after 49200 training steps, the loss is 0.162706, the validation accuracy is 0.965\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 49300 training steps, the loss is 0.15235, the validation accuracy is 0.9582\n",
      "after 49400 training steps, the loss is 0.0878328, the validation accuracy is 0.9632\n",
      "after 49500 training steps, the loss is 0.101938, the validation accuracy is 0.9658\n",
      "after 49600 training steps, the loss is 0.128335, the validation accuracy is 0.9664\n",
      "after 49700 training steps, the loss is 0.0814958, the validation accuracy is 0.9644\n",
      "after 49800 training steps, the loss is 0.0731393, the validation accuracy is 0.969\n",
      "after 49900 training steps, the loss is 0.0729244, the validation accuracy is 0.9678\n",
      "after 50000 training steps, the loss is 0.103478, the validation accuracy is 0.9638\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.96616364\n",
      "the test accuarcy is: 0.961\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=50000, lr=0.001, lambda_flag='l2', lambda_value=0.01, initial_way=MSRA, \n",
    "      hidden1=1000, hidden2=1000, hidden3=1000, hidden4=1000, hidden5=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "太复杂了耗时，而且效果没有那么好，可能是迭代次数不够，把模型再变简单点，同时把batch_size变为64试试～"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 2.15014, the validation accuracy is 0.8502\n",
      "after 200 training steps, the loss is 1.22944, the validation accuracy is 0.9116\n",
      "after 300 training steps, the loss is 0.722463, the validation accuracy is 0.9268\n",
      "after 400 training steps, the loss is 0.69774, the validation accuracy is 0.9344\n",
      "after 500 training steps, the loss is 0.333031, the validation accuracy is 0.9432\n",
      "after 600 training steps, the loss is 0.312454, the validation accuracy is 0.9524\n",
      "after 700 training steps, the loss is 0.359637, the validation accuracy is 0.9506\n",
      "after 800 training steps, the loss is 0.302372, the validation accuracy is 0.9552\n",
      "after 900 training steps, the loss is 0.310518, the validation accuracy is 0.9556\n",
      "after 1000 training steps, the loss is 0.101402, the validation accuracy is 0.9576\n",
      "after 1100 training steps, the loss is 0.125886, the validation accuracy is 0.9578\n",
      "after 1200 training steps, the loss is 0.144971, the validation accuracy is 0.9596\n",
      "after 1300 training steps, the loss is 0.106749, the validation accuracy is 0.9668\n",
      "after 1400 training steps, the loss is 0.167324, the validation accuracy is 0.9618\n",
      "after 1500 training steps, the loss is 0.126449, the validation accuracy is 0.9662\n",
      "after 1600 training steps, the loss is 0.0905827, the validation accuracy is 0.9678\n",
      "after 1700 training steps, the loss is 0.395516, the validation accuracy is 0.961\n",
      "after 1800 training steps, the loss is 0.106082, the validation accuracy is 0.9652\n",
      "after 1900 training steps, the loss is 0.113497, the validation accuracy is 0.9706\n",
      "after 2000 training steps, the loss is 0.122391, the validation accuracy is 0.969\n",
      "after 2100 training steps, the loss is 0.110442, the validation accuracy is 0.9602\n",
      "after 2200 training steps, the loss is 0.184592, the validation accuracy is 0.9684\n",
      "after 2300 training steps, the loss is 0.182667, the validation accuracy is 0.9632\n",
      "after 2400 training steps, the loss is 0.146125, the validation accuracy is 0.9702\n",
      "after 2500 training steps, the loss is 0.115661, the validation accuracy is 0.9656\n",
      "after 2600 training steps, the loss is 0.12939, the validation accuracy is 0.9588\n",
      "after 2700 training steps, the loss is 0.136102, the validation accuracy is 0.9748\n",
      "after 2800 training steps, the loss is 0.105067, the validation accuracy is 0.9742\n",
      "after 2900 training steps, the loss is 0.0567209, the validation accuracy is 0.9748\n",
      "after 3000 training steps, the loss is 0.0595913, the validation accuracy is 0.9766\n",
      "after 3100 training steps, the loss is 0.0789261, the validation accuracy is 0.9738\n",
      "after 3200 training steps, the loss is 0.0688896, the validation accuracy is 0.9704\n",
      "after 3300 training steps, the loss is 0.102199, the validation accuracy is 0.9766\n",
      "after 3400 training steps, the loss is 0.0820573, the validation accuracy is 0.973\n",
      "after 3500 training steps, the loss is 0.0545466, the validation accuracy is 0.9696\n",
      "after 3600 training steps, the loss is 0.141031, the validation accuracy is 0.9744\n",
      "after 3700 training steps, the loss is 0.129289, the validation accuracy is 0.9756\n",
      "after 3800 training steps, the loss is 0.0579394, the validation accuracy is 0.9756\n",
      "after 3900 training steps, the loss is 0.158865, the validation accuracy is 0.9736\n",
      "after 4000 training steps, the loss is 0.0605855, the validation accuracy is 0.9748\n",
      "after 4100 training steps, the loss is 0.149057, the validation accuracy is 0.968\n",
      "after 4200 training steps, the loss is 0.105073, the validation accuracy is 0.979\n",
      "after 4300 training steps, the loss is 0.069742, the validation accuracy is 0.9756\n",
      "after 4400 training steps, the loss is 0.0619649, the validation accuracy is 0.9762\n",
      "after 4500 training steps, the loss is 0.0463527, the validation accuracy is 0.9752\n",
      "after 4600 training steps, the loss is 0.0489065, the validation accuracy is 0.9768\n",
      "after 4700 training steps, the loss is 0.0774551, the validation accuracy is 0.9746\n",
      "after 4800 training steps, the loss is 0.0783305, the validation accuracy is 0.9754\n",
      "after 4900 training steps, the loss is 0.0732009, the validation accuracy is 0.9796\n",
      "after 5000 training steps, the loss is 0.0719752, the validation accuracy is 0.9782\n",
      "after 5100 training steps, the loss is 0.0648801, the validation accuracy is 0.9772\n",
      "after 5200 training steps, the loss is 0.049026, the validation accuracy is 0.9798\n",
      "after 5300 training steps, the loss is 0.0746446, the validation accuracy is 0.9796\n",
      "after 5400 training steps, the loss is 0.0360718, the validation accuracy is 0.977\n",
      "after 5500 training steps, the loss is 0.0650049, the validation accuracy is 0.9782\n",
      "after 5600 training steps, the loss is 0.0812751, the validation accuracy is 0.9772\n",
      "after 5700 training steps, the loss is 0.104049, the validation accuracy is 0.971\n",
      "after 5800 training steps, the loss is 0.0550025, the validation accuracy is 0.9768\n",
      "after 5900 training steps, the loss is 0.103971, the validation accuracy is 0.9762\n",
      "after 6000 training steps, the loss is 0.0733, the validation accuracy is 0.9774\n",
      "after 6100 training steps, the loss is 0.039553, the validation accuracy is 0.9788\n",
      "after 6200 training steps, the loss is 0.0561759, the validation accuracy is 0.9784\n",
      "after 6300 training steps, the loss is 0.0891204, the validation accuracy is 0.9818\n",
      "after 6400 training steps, the loss is 0.0589437, the validation accuracy is 0.9814\n",
      "after 6500 training steps, the loss is 0.0554886, the validation accuracy is 0.98\n",
      "after 6600 training steps, the loss is 0.0641812, the validation accuracy is 0.9796\n",
      "after 6700 training steps, the loss is 0.125542, the validation accuracy is 0.9796\n",
      "after 6800 training steps, the loss is 0.138959, the validation accuracy is 0.9776\n",
      "after 6900 training steps, the loss is 0.0721555, the validation accuracy is 0.978\n",
      "after 7000 training steps, the loss is 0.082214, the validation accuracy is 0.9738\n",
      "after 7100 training steps, the loss is 0.0731079, the validation accuracy is 0.975\n",
      "after 7200 training steps, the loss is 0.0298349, the validation accuracy is 0.979\n",
      "after 7300 training steps, the loss is 0.0541751, the validation accuracy is 0.9814\n",
      "after 7400 training steps, the loss is 0.0497933, the validation accuracy is 0.9806\n",
      "after 7500 training steps, the loss is 0.0471257, the validation accuracy is 0.9792\n",
      "after 7600 training steps, the loss is 0.0883559, the validation accuracy is 0.9794\n",
      "after 7700 training steps, the loss is 0.0820524, the validation accuracy is 0.9798\n",
      "after 7800 training steps, the loss is 0.0435975, the validation accuracy is 0.9798\n",
      "after 7900 training steps, the loss is 0.0545525, the validation accuracy is 0.9802\n",
      "after 8000 training steps, the loss is 0.0487932, the validation accuracy is 0.9812\n",
      "after 8100 training steps, the loss is 0.0424694, the validation accuracy is 0.98\n",
      "after 8200 training steps, the loss is 0.0696675, the validation accuracy is 0.979\n",
      "after 8300 training steps, the loss is 0.0486641, the validation accuracy is 0.9814\n",
      "after 8400 training steps, the loss is 0.0575148, the validation accuracy is 0.9798\n",
      "after 8500 training steps, the loss is 0.163256, the validation accuracy is 0.9764\n",
      "after 8600 training steps, the loss is 0.0423596, the validation accuracy is 0.981\n",
      "after 8700 training steps, the loss is 0.053844, the validation accuracy is 0.9804\n",
      "after 8800 training steps, the loss is 0.0434725, the validation accuracy is 0.9806\n",
      "after 8900 training steps, the loss is 0.0474667, the validation accuracy is 0.9798\n",
      "after 9000 training steps, the loss is 0.0270229, the validation accuracy is 0.9812\n",
      "after 9100 training steps, the loss is 0.0422895, the validation accuracy is 0.982\n",
      "after 9200 training steps, the loss is 0.0427552, the validation accuracy is 0.9798\n",
      "after 9300 training steps, the loss is 0.0645945, the validation accuracy is 0.98\n",
      "after 9400 training steps, the loss is 0.0383552, the validation accuracy is 0.9812\n",
      "after 9500 training steps, the loss is 0.0228523, the validation accuracy is 0.982\n",
      "after 9600 training steps, the loss is 0.0317536, the validation accuracy is 0.9842\n",
      "after 9700 training steps, the loss is 0.0246066, the validation accuracy is 0.983\n",
      "after 9800 training steps, the loss is 0.0329543, the validation accuracy is 0.9814\n",
      "after 9900 training steps, the loss is 0.107221, the validation accuracy is 0.9788\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10000 training steps, the loss is 0.0283803, the validation accuracy is 0.98\n",
      "after 10100 training steps, the loss is 0.0362649, the validation accuracy is 0.9832\n",
      "after 10200 training steps, the loss is 0.0321833, the validation accuracy is 0.9812\n",
      "after 10300 training steps, the loss is 0.024503, the validation accuracy is 0.9814\n",
      "after 10400 training steps, the loss is 0.0343269, the validation accuracy is 0.9818\n",
      "after 10500 training steps, the loss is 0.0474198, the validation accuracy is 0.9814\n",
      "after 10600 training steps, the loss is 0.0313585, the validation accuracy is 0.9818\n",
      "after 10700 training steps, the loss is 0.027629, the validation accuracy is 0.9794\n",
      "after 10800 training steps, the loss is 0.0322536, the validation accuracy is 0.981\n",
      "after 10900 training steps, the loss is 0.0466858, the validation accuracy is 0.9788\n",
      "after 11000 training steps, the loss is 0.0387532, the validation accuracy is 0.9816\n",
      "after 11100 training steps, the loss is 0.046742, the validation accuracy is 0.982\n",
      "after 11200 training steps, the loss is 0.0696336, the validation accuracy is 0.9796\n",
      "after 11300 training steps, the loss is 0.0445621, the validation accuracy is 0.9822\n",
      "after 11400 training steps, the loss is 0.0434115, the validation accuracy is 0.9818\n",
      "after 11500 training steps, the loss is 0.0245155, the validation accuracy is 0.9822\n",
      "after 11600 training steps, the loss is 0.0228272, the validation accuracy is 0.9826\n",
      "after 11700 training steps, the loss is 0.0437803, the validation accuracy is 0.9812\n",
      "after 11800 training steps, the loss is 0.0633392, the validation accuracy is 0.9808\n",
      "after 11900 training steps, the loss is 0.0293675, the validation accuracy is 0.9824\n",
      "after 12000 training steps, the loss is 0.0313923, the validation accuracy is 0.9824\n",
      "after 12100 training steps, the loss is 0.0211889, the validation accuracy is 0.983\n",
      "after 12200 training steps, the loss is 0.0359952, the validation accuracy is 0.9822\n",
      "after 12300 training steps, the loss is 0.0384606, the validation accuracy is 0.9814\n",
      "after 12400 training steps, the loss is 0.0356402, the validation accuracy is 0.9804\n",
      "after 12500 training steps, the loss is 0.0284887, the validation accuracy is 0.981\n",
      "after 12600 training steps, the loss is 0.0603884, the validation accuracy is 0.9792\n",
      "after 12700 training steps, the loss is 0.0404105, the validation accuracy is 0.9828\n",
      "after 12800 training steps, the loss is 0.0284017, the validation accuracy is 0.9796\n",
      "after 12900 training steps, the loss is 0.045244, the validation accuracy is 0.9806\n",
      "after 13000 training steps, the loss is 0.0199143, the validation accuracy is 0.9834\n",
      "after 13100 training steps, the loss is 0.0248125, the validation accuracy is 0.9808\n",
      "after 13200 training steps, the loss is 0.0227578, the validation accuracy is 0.9822\n",
      "after 13300 training steps, the loss is 0.0247697, the validation accuracy is 0.9746\n",
      "after 13400 training steps, the loss is 0.0276574, the validation accuracy is 0.9814\n",
      "after 13500 training steps, the loss is 0.0255067, the validation accuracy is 0.9834\n",
      "after 13600 training steps, the loss is 0.028432, the validation accuracy is 0.9824\n",
      "after 13700 training steps, the loss is 0.0288484, the validation accuracy is 0.9838\n",
      "after 13800 training steps, the loss is 0.0317988, the validation accuracy is 0.9828\n",
      "after 13900 training steps, the loss is 0.0403767, the validation accuracy is 0.9792\n",
      "after 14000 training steps, the loss is 0.0232279, the validation accuracy is 0.9826\n",
      "after 14100 training steps, the loss is 0.0369694, the validation accuracy is 0.9814\n",
      "after 14200 training steps, the loss is 0.0235143, the validation accuracy is 0.9846\n",
      "after 14300 training steps, the loss is 0.0220512, the validation accuracy is 0.982\n",
      "after 14400 training steps, the loss is 0.0356606, the validation accuracy is 0.98\n",
      "after 14500 training steps, the loss is 0.0266855, the validation accuracy is 0.9816\n",
      "after 14600 training steps, the loss is 0.0222039, the validation accuracy is 0.9812\n",
      "after 14700 training steps, the loss is 0.0295116, the validation accuracy is 0.983\n",
      "after 14800 training steps, the loss is 0.0318686, the validation accuracy is 0.985\n",
      "after 14900 training steps, the loss is 0.0507722, the validation accuracy is 0.982\n",
      "after 15000 training steps, the loss is 0.0294084, the validation accuracy is 0.9808\n",
      "after 15100 training steps, the loss is 0.0230539, the validation accuracy is 0.9844\n",
      "after 15200 training steps, the loss is 0.0277753, the validation accuracy is 0.9824\n",
      "after 15300 training steps, the loss is 0.0260064, the validation accuracy is 0.9812\n",
      "after 15400 training steps, the loss is 0.024961, the validation accuracy is 0.9814\n",
      "after 15500 training steps, the loss is 0.0209651, the validation accuracy is 0.9818\n",
      "after 15600 training steps, the loss is 0.0377161, the validation accuracy is 0.9818\n",
      "after 15700 training steps, the loss is 0.0177162, the validation accuracy is 0.9818\n",
      "after 15800 training steps, the loss is 0.0411961, the validation accuracy is 0.9836\n",
      "after 15900 training steps, the loss is 0.034485, the validation accuracy is 0.9828\n",
      "after 16000 training steps, the loss is 0.0344371, the validation accuracy is 0.9822\n",
      "after 16100 training steps, the loss is 0.0431216, the validation accuracy is 0.9796\n",
      "after 16200 training steps, the loss is 0.122707, the validation accuracy is 0.9822\n",
      "after 16300 training steps, the loss is 0.0171968, the validation accuracy is 0.9834\n",
      "after 16400 training steps, the loss is 0.0391719, the validation accuracy is 0.9824\n",
      "after 16500 training steps, the loss is 0.0208253, the validation accuracy is 0.9824\n",
      "after 16600 training steps, the loss is 0.0183493, the validation accuracy is 0.9814\n",
      "after 16700 training steps, the loss is 0.02579, the validation accuracy is 0.9816\n",
      "after 16800 training steps, the loss is 0.0268333, the validation accuracy is 0.9806\n",
      "after 16900 training steps, the loss is 0.0195897, the validation accuracy is 0.9818\n",
      "after 17000 training steps, the loss is 0.0187096, the validation accuracy is 0.9842\n",
      "after 17100 training steps, the loss is 0.0250211, the validation accuracy is 0.9828\n",
      "after 17200 training steps, the loss is 0.0239226, the validation accuracy is 0.9832\n",
      "after 17300 training steps, the loss is 0.0209518, the validation accuracy is 0.9832\n",
      "after 17400 training steps, the loss is 0.0273256, the validation accuracy is 0.9826\n",
      "after 17500 training steps, the loss is 0.0191467, the validation accuracy is 0.984\n",
      "after 17600 training steps, the loss is 0.0479382, the validation accuracy is 0.9826\n",
      "after 17700 training steps, the loss is 0.0170932, the validation accuracy is 0.9832\n",
      "after 17800 training steps, the loss is 0.0226585, the validation accuracy is 0.9828\n",
      "after 17900 training steps, the loss is 0.01561, the validation accuracy is 0.982\n",
      "after 18000 training steps, the loss is 0.0187362, the validation accuracy is 0.9828\n",
      "after 18100 training steps, the loss is 0.0223007, the validation accuracy is 0.9844\n",
      "after 18200 training steps, the loss is 0.0166371, the validation accuracy is 0.9828\n",
      "after 18300 training steps, the loss is 0.0195385, the validation accuracy is 0.9822\n",
      "after 18400 training steps, the loss is 0.0199554, the validation accuracy is 0.983\n",
      "after 18500 training steps, the loss is 0.022025, the validation accuracy is 0.9832\n",
      "after 18600 training steps, the loss is 0.0183158, the validation accuracy is 0.9826\n",
      "after 18700 training steps, the loss is 0.0332399, the validation accuracy is 0.9828\n",
      "after 18800 training steps, the loss is 0.0249171, the validation accuracy is 0.9834\n",
      "after 18900 training steps, the loss is 0.0190878, the validation accuracy is 0.9834\n",
      "after 19000 training steps, the loss is 0.0220476, the validation accuracy is 0.9818\n",
      "after 19100 training steps, the loss is 0.0220813, the validation accuracy is 0.9828\n",
      "after 19200 training steps, the loss is 0.0166197, the validation accuracy is 0.9836\n",
      "after 19300 training steps, the loss is 0.0336955, the validation accuracy is 0.979\n",
      "after 19400 training steps, the loss is 0.0250958, the validation accuracy is 0.9822\n",
      "after 19500 training steps, the loss is 0.020754, the validation accuracy is 0.9836\n",
      "after 19600 training steps, the loss is 0.0189417, the validation accuracy is 0.9826\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19700 training steps, the loss is 0.0253468, the validation accuracy is 0.9832\n",
      "after 19800 training steps, the loss is 0.0615955, the validation accuracy is 0.9784\n",
      "after 19900 training steps, the loss is 0.0245746, the validation accuracy is 0.9834\n",
      "after 20000 training steps, the loss is 0.0148925, the validation accuracy is 0.983\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.9991636\n",
      "the test accuarcy is: 0.982\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=64, training_step=20000, lr=0.3, lambda_flag='l2', lambda_value=0.01, initial_way=MSRA, \n",
    "      hidden1=1000, hidden2=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "果然效果上来了，再把迭代次数增加～"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 3.81469, the validation accuracy is 0.8458\n",
      "after 200 training steps, the loss is 2.22654, the validation accuracy is 0.9032\n",
      "after 300 training steps, the loss is 1.30482, the validation accuracy is 0.9194\n",
      "after 400 training steps, the loss is 0.835966, the validation accuracy is 0.931\n",
      "after 500 training steps, the loss is 0.619298, the validation accuracy is 0.9134\n",
      "after 600 training steps, the loss is 0.495093, the validation accuracy is 0.9336\n",
      "after 700 training steps, the loss is 0.270192, the validation accuracy is 0.9524\n",
      "after 800 training steps, the loss is 0.402812, the validation accuracy is 0.9514\n",
      "after 900 training steps, the loss is 0.316788, the validation accuracy is 0.9572\n",
      "after 1000 training steps, the loss is 0.240577, the validation accuracy is 0.9618\n",
      "after 1100 training steps, the loss is 0.182912, the validation accuracy is 0.9596\n",
      "after 1200 training steps, the loss is 0.398093, the validation accuracy is 0.957\n",
      "after 1300 training steps, the loss is 0.176276, the validation accuracy is 0.957\n",
      "after 1400 training steps, the loss is 0.233998, the validation accuracy is 0.966\n",
      "after 1500 training steps, the loss is 0.0695468, the validation accuracy is 0.9658\n",
      "after 1600 training steps, the loss is 0.154237, the validation accuracy is 0.9694\n",
      "after 1700 training steps, the loss is 0.111686, the validation accuracy is 0.9682\n",
      "after 1800 training steps, the loss is 0.124966, the validation accuracy is 0.9712\n",
      "after 1900 training steps, the loss is 0.157796, the validation accuracy is 0.9636\n",
      "after 2000 training steps, the loss is 0.0614519, the validation accuracy is 0.9686\n",
      "after 2100 training steps, the loss is 0.0808113, the validation accuracy is 0.9682\n",
      "after 2200 training steps, the loss is 0.10667, the validation accuracy is 0.9742\n",
      "after 2300 training steps, the loss is 0.060724, the validation accuracy is 0.9742\n",
      "after 2400 training steps, the loss is 0.164024, the validation accuracy is 0.9654\n",
      "after 2500 training steps, the loss is 0.160713, the validation accuracy is 0.9708\n",
      "after 2600 training steps, the loss is 0.11205, the validation accuracy is 0.9732\n",
      "after 2700 training steps, the loss is 0.122063, the validation accuracy is 0.9714\n",
      "after 2800 training steps, the loss is 0.0829536, the validation accuracy is 0.9712\n",
      "after 2900 training steps, the loss is 0.11603, the validation accuracy is 0.9736\n",
      "after 3000 training steps, the loss is 0.113796, the validation accuracy is 0.9702\n",
      "after 3100 training steps, the loss is 0.0812118, the validation accuracy is 0.9748\n",
      "after 3200 training steps, the loss is 0.0745025, the validation accuracy is 0.9758\n",
      "after 3300 training steps, the loss is 0.131001, the validation accuracy is 0.974\n",
      "after 3400 training steps, the loss is 0.0602365, the validation accuracy is 0.975\n",
      "after 3500 training steps, the loss is 0.115883, the validation accuracy is 0.9758\n",
      "after 3600 training steps, the loss is 0.0959896, the validation accuracy is 0.973\n",
      "after 3700 training steps, the loss is 0.0855724, the validation accuracy is 0.9724\n",
      "after 3800 training steps, the loss is 0.0587857, the validation accuracy is 0.9766\n",
      "after 3900 training steps, the loss is 0.0587855, the validation accuracy is 0.976\n",
      "after 4000 training steps, the loss is 0.0379718, the validation accuracy is 0.9734\n",
      "after 4100 training steps, the loss is 0.0764523, the validation accuracy is 0.9748\n",
      "after 4200 training steps, the loss is 0.0873619, the validation accuracy is 0.977\n",
      "after 4300 training steps, the loss is 0.0993538, the validation accuracy is 0.976\n",
      "after 4400 training steps, the loss is 0.076723, the validation accuracy is 0.9754\n",
      "after 4500 training steps, the loss is 0.0591218, the validation accuracy is 0.977\n",
      "after 4600 training steps, the loss is 0.0732218, the validation accuracy is 0.9774\n",
      "after 4700 training steps, the loss is 0.0407512, the validation accuracy is 0.9762\n",
      "after 4800 training steps, the loss is 0.146699, the validation accuracy is 0.9762\n",
      "after 4900 training steps, the loss is 0.0668367, the validation accuracy is 0.9756\n",
      "after 5000 training steps, the loss is 0.0918678, the validation accuracy is 0.9778\n",
      "after 5100 training steps, the loss is 0.0892047, the validation accuracy is 0.9704\n",
      "after 5200 training steps, the loss is 0.112569, the validation accuracy is 0.9778\n",
      "after 5300 training steps, the loss is 0.0444767, the validation accuracy is 0.9776\n",
      "after 5400 training steps, the loss is 0.0949458, the validation accuracy is 0.9764\n",
      "after 5500 training steps, the loss is 0.053654, the validation accuracy is 0.9788\n",
      "after 5600 training steps, the loss is 0.11618, the validation accuracy is 0.9784\n",
      "after 5700 training steps, the loss is 0.0472427, the validation accuracy is 0.9766\n",
      "after 5800 training steps, the loss is 0.0310498, the validation accuracy is 0.979\n",
      "after 5900 training steps, the loss is 0.055496, the validation accuracy is 0.9778\n",
      "after 6000 training steps, the loss is 0.0671438, the validation accuracy is 0.9774\n",
      "after 6100 training steps, the loss is 0.0468573, the validation accuracy is 0.9782\n",
      "after 6200 training steps, the loss is 0.058625, the validation accuracy is 0.978\n",
      "after 6300 training steps, the loss is 0.0468224, the validation accuracy is 0.98\n",
      "after 6400 training steps, the loss is 0.115703, the validation accuracy is 0.9788\n",
      "after 6500 training steps, the loss is 0.0421514, the validation accuracy is 0.9788\n",
      "after 6600 training steps, the loss is 0.137858, the validation accuracy is 0.9774\n",
      "after 6700 training steps, the loss is 0.0803857, the validation accuracy is 0.9782\n",
      "after 6800 training steps, the loss is 0.0635663, the validation accuracy is 0.9796\n",
      "after 6900 training steps, the loss is 0.0577965, the validation accuracy is 0.9782\n",
      "after 7000 training steps, the loss is 0.0602115, the validation accuracy is 0.9804\n",
      "after 7100 training steps, the loss is 0.0945854, the validation accuracy is 0.9786\n",
      "after 7200 training steps, the loss is 0.0669012, the validation accuracy is 0.9806\n",
      "after 7300 training steps, the loss is 0.0801507, the validation accuracy is 0.9716\n",
      "after 7400 training steps, the loss is 0.0370465, the validation accuracy is 0.9794\n",
      "after 7500 training steps, the loss is 0.031483, the validation accuracy is 0.9802\n",
      "after 7600 training steps, the loss is 0.0611975, the validation accuracy is 0.9798\n",
      "after 7700 training steps, the loss is 0.0413158, the validation accuracy is 0.9812\n",
      "after 7800 training steps, the loss is 0.0523335, the validation accuracy is 0.9786\n",
      "after 7900 training steps, the loss is 0.0344828, the validation accuracy is 0.9808\n",
      "after 8000 training steps, the loss is 0.0250685, the validation accuracy is 0.981\n",
      "after 8100 training steps, the loss is 0.0984343, the validation accuracy is 0.9792\n",
      "after 8200 training steps, the loss is 0.0400743, the validation accuracy is 0.9804\n",
      "after 8300 training steps, the loss is 0.0300508, the validation accuracy is 0.9802\n",
      "after 8400 training steps, the loss is 0.0382046, the validation accuracy is 0.981\n",
      "after 8500 training steps, the loss is 0.0702863, the validation accuracy is 0.9798\n",
      "after 8600 training steps, the loss is 0.0329565, the validation accuracy is 0.9812\n",
      "after 8700 training steps, the loss is 0.0639673, the validation accuracy is 0.9762\n",
      "after 8800 training steps, the loss is 0.0386839, the validation accuracy is 0.9812\n",
      "after 8900 training steps, the loss is 0.0268952, the validation accuracy is 0.9784\n",
      "after 9000 training steps, the loss is 0.0534395, the validation accuracy is 0.9816\n",
      "after 9100 training steps, the loss is 0.0646878, the validation accuracy is 0.9784\n",
      "after 9200 training steps, the loss is 0.0345013, the validation accuracy is 0.9816\n",
      "after 9300 training steps, the loss is 0.0515467, the validation accuracy is 0.9818\n",
      "after 9400 training steps, the loss is 0.117467, the validation accuracy is 0.9772\n",
      "after 9500 training steps, the loss is 0.0292426, the validation accuracy is 0.9816\n",
      "after 9600 training steps, the loss is 0.0371814, the validation accuracy is 0.9782\n",
      "after 9700 training steps, the loss is 0.0347856, the validation accuracy is 0.9792\n",
      "after 9800 training steps, the loss is 0.054914, the validation accuracy is 0.9814\n",
      "after 9900 training steps, the loss is 0.0650866, the validation accuracy is 0.9808\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10000 training steps, the loss is 0.0301274, the validation accuracy is 0.98\n",
      "after 10100 training steps, the loss is 0.0262996, the validation accuracy is 0.9806\n",
      "after 10200 training steps, the loss is 0.0357755, the validation accuracy is 0.9808\n",
      "after 10300 training steps, the loss is 0.0509681, the validation accuracy is 0.9816\n",
      "after 10400 training steps, the loss is 0.0376231, the validation accuracy is 0.9824\n",
      "after 10500 training steps, the loss is 0.0709163, the validation accuracy is 0.9764\n",
      "after 10600 training steps, the loss is 0.0495268, the validation accuracy is 0.983\n",
      "after 10700 training steps, the loss is 0.0209244, the validation accuracy is 0.9826\n",
      "after 10800 training steps, the loss is 0.031181, the validation accuracy is 0.981\n",
      "after 10900 training steps, the loss is 0.0569158, the validation accuracy is 0.9782\n",
      "after 11000 training steps, the loss is 0.0302686, the validation accuracy is 0.982\n",
      "after 11100 training steps, the loss is 0.0537545, the validation accuracy is 0.9798\n",
      "after 11200 training steps, the loss is 0.0459774, the validation accuracy is 0.9818\n",
      "after 11300 training steps, the loss is 0.0354131, the validation accuracy is 0.9824\n",
      "after 11400 training steps, the loss is 0.0274255, the validation accuracy is 0.9796\n",
      "after 11500 training steps, the loss is 0.0380245, the validation accuracy is 0.9804\n",
      "after 11600 training steps, the loss is 0.0434932, the validation accuracy is 0.981\n",
      "after 11700 training steps, the loss is 0.0435174, the validation accuracy is 0.982\n",
      "after 11800 training steps, the loss is 0.0219979, the validation accuracy is 0.9818\n",
      "after 11900 training steps, the loss is 0.0894217, the validation accuracy is 0.9818\n",
      "after 12000 training steps, the loss is 0.0278335, the validation accuracy is 0.9832\n",
      "after 12100 training steps, the loss is 0.0301848, the validation accuracy is 0.9796\n",
      "after 12200 training steps, the loss is 0.0361721, the validation accuracy is 0.98\n",
      "after 12300 training steps, the loss is 0.0353998, the validation accuracy is 0.98\n",
      "after 12400 training steps, the loss is 0.0264434, the validation accuracy is 0.981\n",
      "after 12500 training steps, the loss is 0.025435, the validation accuracy is 0.9824\n",
      "after 12600 training steps, the loss is 0.0662047, the validation accuracy is 0.9818\n",
      "after 12700 training steps, the loss is 0.0336801, the validation accuracy is 0.981\n",
      "after 12800 training steps, the loss is 0.0779192, the validation accuracy is 0.9786\n",
      "after 12900 training steps, the loss is 0.0500942, the validation accuracy is 0.9786\n",
      "after 13000 training steps, the loss is 0.0265526, the validation accuracy is 0.9818\n",
      "after 13100 training steps, the loss is 0.0432352, the validation accuracy is 0.9806\n",
      "after 13200 training steps, the loss is 0.0897717, the validation accuracy is 0.9802\n",
      "after 13300 training steps, the loss is 0.0246163, the validation accuracy is 0.9826\n",
      "after 13400 training steps, the loss is 0.122132, the validation accuracy is 0.9812\n",
      "after 13500 training steps, the loss is 0.0218887, the validation accuracy is 0.9826\n",
      "after 13600 training steps, the loss is 0.0241343, the validation accuracy is 0.9822\n",
      "after 13700 training steps, the loss is 0.0264317, the validation accuracy is 0.9808\n",
      "after 13800 training steps, the loss is 0.0262913, the validation accuracy is 0.9814\n",
      "after 13900 training steps, the loss is 0.0364894, the validation accuracy is 0.9834\n",
      "after 14000 training steps, the loss is 0.0359958, the validation accuracy is 0.9834\n",
      "after 14100 training steps, the loss is 0.0507697, the validation accuracy is 0.9794\n",
      "after 14200 training steps, the loss is 0.0240579, the validation accuracy is 0.9822\n",
      "after 14300 training steps, the loss is 0.0195354, the validation accuracy is 0.9834\n",
      "after 14400 training steps, the loss is 0.0307029, the validation accuracy is 0.9846\n",
      "after 14500 training steps, the loss is 0.0254471, the validation accuracy is 0.9842\n",
      "after 14600 training steps, the loss is 0.0195468, the validation accuracy is 0.983\n",
      "after 14700 training steps, the loss is 0.0419682, the validation accuracy is 0.9832\n",
      "after 14800 training steps, the loss is 0.0319354, the validation accuracy is 0.9816\n",
      "after 14900 training steps, the loss is 0.0399563, the validation accuracy is 0.9822\n",
      "after 15000 training steps, the loss is 0.0349675, the validation accuracy is 0.9792\n",
      "after 15100 training steps, the loss is 0.0346585, the validation accuracy is 0.983\n",
      "after 15200 training steps, the loss is 0.0230392, the validation accuracy is 0.9832\n",
      "after 15300 training steps, the loss is 0.0272081, the validation accuracy is 0.9824\n",
      "after 15400 training steps, the loss is 0.0219168, the validation accuracy is 0.982\n",
      "after 15500 training steps, the loss is 0.0311548, the validation accuracy is 0.9826\n",
      "after 15600 training steps, the loss is 0.0225765, the validation accuracy is 0.9838\n",
      "after 15700 training steps, the loss is 0.0262849, the validation accuracy is 0.9832\n",
      "after 15800 training steps, the loss is 0.0242159, the validation accuracy is 0.9828\n",
      "after 15900 training steps, the loss is 0.0450145, the validation accuracy is 0.9834\n",
      "after 16000 training steps, the loss is 0.0279204, the validation accuracy is 0.9822\n",
      "after 16100 training steps, the loss is 0.0256915, the validation accuracy is 0.9828\n",
      "after 16200 training steps, the loss is 0.0196924, the validation accuracy is 0.9836\n",
      "after 16300 training steps, the loss is 0.0274955, the validation accuracy is 0.9814\n",
      "after 16400 training steps, the loss is 0.0344544, the validation accuracy is 0.9828\n",
      "after 16500 training steps, the loss is 0.0432327, the validation accuracy is 0.9836\n",
      "after 16600 training steps, the loss is 0.0301793, the validation accuracy is 0.9824\n",
      "after 16700 training steps, the loss is 0.0201589, the validation accuracy is 0.983\n",
      "after 16800 training steps, the loss is 0.0169234, the validation accuracy is 0.9838\n",
      "after 16900 training steps, the loss is 0.0231807, the validation accuracy is 0.983\n",
      "after 17000 training steps, the loss is 0.030983, the validation accuracy is 0.9822\n",
      "after 17100 training steps, the loss is 0.0237885, the validation accuracy is 0.9814\n",
      "after 17200 training steps, the loss is 0.0291367, the validation accuracy is 0.983\n",
      "after 17300 training steps, the loss is 0.0263762, the validation accuracy is 0.983\n",
      "after 17400 training steps, the loss is 0.035945, the validation accuracy is 0.9824\n",
      "after 17500 training steps, the loss is 0.0262356, the validation accuracy is 0.9822\n",
      "after 17600 training steps, the loss is 0.024545, the validation accuracy is 0.9828\n",
      "after 17700 training steps, the loss is 0.0217878, the validation accuracy is 0.9828\n",
      "after 17800 training steps, the loss is 0.0213239, the validation accuracy is 0.9826\n",
      "after 17900 training steps, the loss is 0.0156003, the validation accuracy is 0.9838\n",
      "after 18000 training steps, the loss is 0.0189391, the validation accuracy is 0.982\n",
      "after 18100 training steps, the loss is 0.0222807, the validation accuracy is 0.9832\n",
      "after 18200 training steps, the loss is 0.023172, the validation accuracy is 0.9836\n",
      "after 18300 training steps, the loss is 0.0333556, the validation accuracy is 0.9824\n",
      "after 18400 training steps, the loss is 0.020735, the validation accuracy is 0.982\n",
      "after 18500 training steps, the loss is 0.0568284, the validation accuracy is 0.979\n",
      "after 18600 training steps, the loss is 0.0201512, the validation accuracy is 0.984\n",
      "after 18700 training steps, the loss is 0.0200397, the validation accuracy is 0.9822\n",
      "after 18800 training steps, the loss is 0.0254328, the validation accuracy is 0.9832\n",
      "after 18900 training steps, the loss is 0.0520456, the validation accuracy is 0.9802\n",
      "after 19000 training steps, the loss is 0.0203207, the validation accuracy is 0.9828\n",
      "after 19100 training steps, the loss is 0.0162973, the validation accuracy is 0.9846\n",
      "after 19200 training steps, the loss is 0.0174556, the validation accuracy is 0.9844\n",
      "after 19300 training steps, the loss is 0.0162754, the validation accuracy is 0.9818\n",
      "after 19400 training steps, the loss is 0.0145402, the validation accuracy is 0.984\n",
      "after 19500 training steps, the loss is 0.0189113, the validation accuracy is 0.9828\n",
      "after 19600 training steps, the loss is 0.0181321, the validation accuracy is 0.9824\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19700 training steps, the loss is 0.0192485, the validation accuracy is 0.9828\n",
      "after 19800 training steps, the loss is 0.0305333, the validation accuracy is 0.9848\n",
      "after 19900 training steps, the loss is 0.0197762, the validation accuracy is 0.9848\n",
      "after 20000 training steps, the loss is 0.0190761, the validation accuracy is 0.9838\n",
      "after 20100 training steps, the loss is 0.0187205, the validation accuracy is 0.983\n",
      "after 20200 training steps, the loss is 0.0214776, the validation accuracy is 0.9834\n",
      "after 20300 training steps, the loss is 0.0144813, the validation accuracy is 0.9848\n",
      "after 20400 training steps, the loss is 0.0169134, the validation accuracy is 0.9844\n",
      "after 20500 training steps, the loss is 0.0247054, the validation accuracy is 0.9828\n",
      "after 20600 training steps, the loss is 0.0143691, the validation accuracy is 0.9838\n",
      "after 20700 training steps, the loss is 0.0285095, the validation accuracy is 0.9842\n",
      "after 20800 training steps, the loss is 0.0208342, the validation accuracy is 0.984\n",
      "after 20900 training steps, the loss is 0.0808457, the validation accuracy is 0.9834\n",
      "after 21000 training steps, the loss is 0.0211275, the validation accuracy is 0.9856\n",
      "after 21100 training steps, the loss is 0.0349392, the validation accuracy is 0.9822\n",
      "after 21200 training steps, the loss is 0.0196653, the validation accuracy is 0.9824\n",
      "after 21300 training steps, the loss is 0.0223478, the validation accuracy is 0.9834\n",
      "after 21400 training steps, the loss is 0.0150841, the validation accuracy is 0.9838\n",
      "after 21500 training steps, the loss is 0.0218973, the validation accuracy is 0.9832\n",
      "after 21600 training steps, the loss is 0.0221538, the validation accuracy is 0.9844\n",
      "after 21700 training steps, the loss is 0.0260024, the validation accuracy is 0.9828\n",
      "after 21800 training steps, the loss is 0.019739, the validation accuracy is 0.9852\n",
      "after 21900 training steps, the loss is 0.0322664, the validation accuracy is 0.982\n",
      "after 22000 training steps, the loss is 0.0299439, the validation accuracy is 0.9828\n",
      "after 22100 training steps, the loss is 0.0237992, the validation accuracy is 0.9854\n",
      "after 22200 training steps, the loss is 0.0305165, the validation accuracy is 0.9848\n",
      "after 22300 training steps, the loss is 0.016055, the validation accuracy is 0.9842\n",
      "after 22400 training steps, the loss is 0.0166442, the validation accuracy is 0.9826\n",
      "after 22500 training steps, the loss is 0.0155486, the validation accuracy is 0.9834\n",
      "after 22600 training steps, the loss is 0.0184406, the validation accuracy is 0.9828\n",
      "after 22700 training steps, the loss is 0.015298, the validation accuracy is 0.9844\n",
      "after 22800 training steps, the loss is 0.0164212, the validation accuracy is 0.9842\n",
      "after 22900 training steps, the loss is 0.0197113, the validation accuracy is 0.9834\n",
      "after 23000 training steps, the loss is 0.043145, the validation accuracy is 0.9816\n",
      "after 23100 training steps, the loss is 0.0171285, the validation accuracy is 0.983\n",
      "after 23200 training steps, the loss is 0.0159952, the validation accuracy is 0.9838\n",
      "after 23300 training steps, the loss is 0.0220873, the validation accuracy is 0.9834\n",
      "after 23400 training steps, the loss is 0.0156121, the validation accuracy is 0.9834\n",
      "after 23500 training steps, the loss is 0.0135445, the validation accuracy is 0.9848\n",
      "after 23600 training steps, the loss is 0.0205138, the validation accuracy is 0.984\n",
      "after 23700 training steps, the loss is 0.0922768, the validation accuracy is 0.9842\n",
      "after 23800 training steps, the loss is 0.0173239, the validation accuracy is 0.9848\n",
      "after 23900 training steps, the loss is 0.0486745, the validation accuracy is 0.9812\n",
      "after 24000 training steps, the loss is 0.0864816, the validation accuracy is 0.9826\n",
      "after 24100 training steps, the loss is 0.0175072, the validation accuracy is 0.9838\n",
      "after 24200 training steps, the loss is 0.0190021, the validation accuracy is 0.984\n",
      "after 24300 training steps, the loss is 0.0205035, the validation accuracy is 0.9842\n",
      "after 24400 training steps, the loss is 0.0185256, the validation accuracy is 0.985\n",
      "after 24500 training steps, the loss is 0.0168956, the validation accuracy is 0.9848\n",
      "after 24600 training steps, the loss is 0.0212484, the validation accuracy is 0.9838\n",
      "after 24700 training steps, the loss is 0.0186329, the validation accuracy is 0.9836\n",
      "after 24800 training steps, the loss is 0.0146055, the validation accuracy is 0.9844\n",
      "after 24900 training steps, the loss is 0.0138098, the validation accuracy is 0.9846\n",
      "after 25000 training steps, the loss is 0.0368942, the validation accuracy is 0.9798\n",
      "after 25100 training steps, the loss is 0.0250281, the validation accuracy is 0.9828\n",
      "after 25200 training steps, the loss is 0.028474, the validation accuracy is 0.982\n",
      "after 25300 training steps, the loss is 0.01555, the validation accuracy is 0.983\n",
      "after 25400 training steps, the loss is 0.0244843, the validation accuracy is 0.9848\n",
      "after 25500 training steps, the loss is 0.0135155, the validation accuracy is 0.9828\n",
      "after 25600 training steps, the loss is 0.0173085, the validation accuracy is 0.9852\n",
      "after 25700 training steps, the loss is 0.0132827, the validation accuracy is 0.9834\n",
      "after 25800 training steps, the loss is 0.0343283, the validation accuracy is 0.9822\n",
      "after 25900 training steps, the loss is 0.0223404, the validation accuracy is 0.9832\n",
      "after 26000 training steps, the loss is 0.0159532, the validation accuracy is 0.985\n",
      "after 26100 training steps, the loss is 0.0129457, the validation accuracy is 0.982\n",
      "after 26200 training steps, the loss is 0.018766, the validation accuracy is 0.9826\n",
      "after 26300 training steps, the loss is 0.0234464, the validation accuracy is 0.9844\n",
      "after 26400 training steps, the loss is 0.0117078, the validation accuracy is 0.9828\n",
      "after 26500 training steps, the loss is 0.0155104, the validation accuracy is 0.9838\n",
      "after 26600 training steps, the loss is 0.0158618, the validation accuracy is 0.9844\n",
      "after 26700 training steps, the loss is 0.0160128, the validation accuracy is 0.9834\n",
      "after 26800 training steps, the loss is 0.0336607, the validation accuracy is 0.9792\n",
      "after 26900 training steps, the loss is 0.014966, the validation accuracy is 0.9836\n",
      "after 27000 training steps, the loss is 0.0155003, the validation accuracy is 0.984\n",
      "after 27100 training steps, the loss is 0.013625, the validation accuracy is 0.9846\n",
      "after 27200 training steps, the loss is 0.0211808, the validation accuracy is 0.985\n",
      "after 27300 training steps, the loss is 0.0136055, the validation accuracy is 0.9842\n",
      "after 27400 training steps, the loss is 0.0172613, the validation accuracy is 0.9842\n",
      "after 27500 training steps, the loss is 0.0245789, the validation accuracy is 0.986\n",
      "after 27600 training steps, the loss is 0.0147028, the validation accuracy is 0.9828\n",
      "after 27700 training steps, the loss is 0.0133278, the validation accuracy is 0.9848\n",
      "after 27800 training steps, the loss is 0.0176211, the validation accuracy is 0.9838\n",
      "after 27900 training steps, the loss is 0.0190838, the validation accuracy is 0.9834\n",
      "after 28000 training steps, the loss is 0.0169782, the validation accuracy is 0.9852\n",
      "after 28100 training steps, the loss is 0.0130614, the validation accuracy is 0.9844\n",
      "after 28200 training steps, the loss is 0.0145929, the validation accuracy is 0.9842\n",
      "after 28300 training steps, the loss is 0.0210204, the validation accuracy is 0.9832\n",
      "after 28400 training steps, the loss is 0.0115701, the validation accuracy is 0.9834\n",
      "after 28500 training steps, the loss is 0.0138371, the validation accuracy is 0.9822\n",
      "after 28600 training steps, the loss is 0.0117765, the validation accuracy is 0.9844\n",
      "after 28700 training steps, the loss is 0.0174522, the validation accuracy is 0.9826\n",
      "after 28800 training steps, the loss is 0.0119691, the validation accuracy is 0.984\n",
      "after 28900 training steps, the loss is 0.0168163, the validation accuracy is 0.9838\n",
      "after 29000 training steps, the loss is 0.0119007, the validation accuracy is 0.9842\n",
      "after 29100 training steps, the loss is 0.0154738, the validation accuracy is 0.9844\n",
      "after 29200 training steps, the loss is 0.0135912, the validation accuracy is 0.9866\n",
      "after 29300 training steps, the loss is 0.025911, the validation accuracy is 0.9838\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 29400 training steps, the loss is 0.0208931, the validation accuracy is 0.9832\n",
      "after 29500 training steps, the loss is 0.0204102, the validation accuracy is 0.983\n",
      "after 29600 training steps, the loss is 0.0139248, the validation accuracy is 0.9832\n",
      "after 29700 training steps, the loss is 0.0185647, the validation accuracy is 0.9834\n",
      "after 29800 training steps, the loss is 0.0158581, the validation accuracy is 0.9834\n",
      "after 29900 training steps, the loss is 0.0197291, the validation accuracy is 0.9846\n",
      "after 30000 training steps, the loss is 0.0157897, the validation accuracy is 0.9846\n",
      "after 30100 training steps, the loss is 0.014487, the validation accuracy is 0.9832\n",
      "after 30200 training steps, the loss is 0.012605, the validation accuracy is 0.9842\n",
      "after 30300 training steps, the loss is 0.0148414, the validation accuracy is 0.9834\n",
      "after 30400 training steps, the loss is 0.0187488, the validation accuracy is 0.9832\n",
      "after 30500 training steps, the loss is 0.0144366, the validation accuracy is 0.9856\n",
      "after 30600 training steps, the loss is 0.0180188, the validation accuracy is 0.9846\n",
      "after 30700 training steps, the loss is 0.0127341, the validation accuracy is 0.9828\n",
      "after 30800 training steps, the loss is 0.0131556, the validation accuracy is 0.9842\n",
      "after 30900 training steps, the loss is 0.0261865, the validation accuracy is 0.9804\n",
      "after 31000 training steps, the loss is 0.0131035, the validation accuracy is 0.9846\n",
      "after 31100 training steps, the loss is 0.0165944, the validation accuracy is 0.9832\n",
      "after 31200 training steps, the loss is 0.0147547, the validation accuracy is 0.9842\n",
      "after 31300 training steps, the loss is 0.0177162, the validation accuracy is 0.9854\n",
      "after 31400 training steps, the loss is 0.0143298, the validation accuracy is 0.9844\n",
      "after 31500 training steps, the loss is 0.0260213, the validation accuracy is 0.9822\n",
      "after 31600 training steps, the loss is 0.013883, the validation accuracy is 0.9854\n",
      "after 31700 training steps, the loss is 0.0178457, the validation accuracy is 0.985\n",
      "after 31800 training steps, the loss is 0.0130675, the validation accuracy is 0.9836\n",
      "after 31900 training steps, the loss is 0.0126981, the validation accuracy is 0.984\n",
      "after 32000 training steps, the loss is 0.01256, the validation accuracy is 0.9832\n",
      "after 32100 training steps, the loss is 0.0163262, the validation accuracy is 0.984\n",
      "after 32200 training steps, the loss is 0.0127824, the validation accuracy is 0.9852\n",
      "after 32300 training steps, the loss is 0.0121004, the validation accuracy is 0.983\n",
      "after 32400 training steps, the loss is 0.0173544, the validation accuracy is 0.9836\n",
      "after 32500 training steps, the loss is 0.0134971, the validation accuracy is 0.9836\n",
      "after 32600 training steps, the loss is 0.0200605, the validation accuracy is 0.9834\n",
      "after 32700 training steps, the loss is 0.0148186, the validation accuracy is 0.9848\n",
      "after 32800 training steps, the loss is 0.0177037, the validation accuracy is 0.9828\n",
      "after 32900 training steps, the loss is 0.0179588, the validation accuracy is 0.986\n",
      "after 33000 training steps, the loss is 0.0198258, the validation accuracy is 0.9836\n",
      "after 33100 training steps, the loss is 0.0216437, the validation accuracy is 0.983\n",
      "after 33200 training steps, the loss is 0.0163378, the validation accuracy is 0.9842\n",
      "after 33300 training steps, the loss is 0.0117285, the validation accuracy is 0.9848\n",
      "after 33400 training steps, the loss is 0.0161357, the validation accuracy is 0.9838\n",
      "after 33500 training steps, the loss is 0.0139355, the validation accuracy is 0.9848\n",
      "after 33600 training steps, the loss is 0.0164259, the validation accuracy is 0.9836\n",
      "after 33700 training steps, the loss is 0.00976169, the validation accuracy is 0.9856\n",
      "after 33800 training steps, the loss is 0.0136518, the validation accuracy is 0.9844\n",
      "after 33900 training steps, the loss is 0.0107287, the validation accuracy is 0.9822\n",
      "after 34000 training steps, the loss is 0.0106595, the validation accuracy is 0.985\n",
      "after 34100 training steps, the loss is 0.011798, the validation accuracy is 0.9846\n",
      "after 34200 training steps, the loss is 0.0126399, the validation accuracy is 0.9852\n",
      "after 34300 training steps, the loss is 0.0146444, the validation accuracy is 0.9856\n",
      "after 34400 training steps, the loss is 0.0144443, the validation accuracy is 0.9848\n",
      "after 34500 training steps, the loss is 0.0163505, the validation accuracy is 0.9838\n",
      "after 34600 training steps, the loss is 0.0180624, the validation accuracy is 0.9822\n",
      "after 34700 training steps, the loss is 0.0160653, the validation accuracy is 0.9848\n",
      "after 34800 training steps, the loss is 0.0120848, the validation accuracy is 0.9852\n",
      "after 34900 training steps, the loss is 0.0147565, the validation accuracy is 0.9846\n",
      "after 35000 training steps, the loss is 0.0124466, the validation accuracy is 0.9852\n",
      "after 35100 training steps, the loss is 0.0107277, the validation accuracy is 0.9844\n",
      "after 35200 training steps, the loss is 0.0101092, the validation accuracy is 0.9832\n",
      "after 35300 training steps, the loss is 0.0116868, the validation accuracy is 0.9852\n",
      "after 35400 training steps, the loss is 0.013254, the validation accuracy is 0.9844\n",
      "after 35500 training steps, the loss is 0.0104667, the validation accuracy is 0.9842\n",
      "after 35600 training steps, the loss is 0.0125032, the validation accuracy is 0.9846\n",
      "after 35700 training steps, the loss is 0.0133693, the validation accuracy is 0.9844\n",
      "after 35800 training steps, the loss is 0.0429956, the validation accuracy is 0.9766\n",
      "after 35900 training steps, the loss is 0.0110104, the validation accuracy is 0.985\n",
      "after 36000 training steps, the loss is 0.0123812, the validation accuracy is 0.9852\n",
      "after 36100 training steps, the loss is 0.0126096, the validation accuracy is 0.985\n",
      "after 36200 training steps, the loss is 0.012841, the validation accuracy is 0.9836\n",
      "after 36300 training steps, the loss is 0.0102695, the validation accuracy is 0.985\n",
      "after 36400 training steps, the loss is 0.0120678, the validation accuracy is 0.9854\n",
      "after 36500 training steps, the loss is 0.0109229, the validation accuracy is 0.9856\n",
      "after 36600 training steps, the loss is 0.0126967, the validation accuracy is 0.9854\n",
      "after 36700 training steps, the loss is 0.0142335, the validation accuracy is 0.984\n",
      "after 36800 training steps, the loss is 0.0108651, the validation accuracy is 0.9858\n",
      "after 36900 training steps, the loss is 0.0095719, the validation accuracy is 0.9842\n",
      "after 37000 training steps, the loss is 0.0140476, the validation accuracy is 0.9844\n",
      "after 37100 training steps, the loss is 0.0110809, the validation accuracy is 0.984\n",
      "after 37200 training steps, the loss is 0.0188643, the validation accuracy is 0.984\n",
      "after 37300 training steps, the loss is 0.0137224, the validation accuracy is 0.9842\n",
      "after 37400 training steps, the loss is 0.0116435, the validation accuracy is 0.9856\n",
      "after 37500 training steps, the loss is 0.0106918, the validation accuracy is 0.9846\n",
      "after 37600 training steps, the loss is 0.0116227, the validation accuracy is 0.9846\n",
      "after 37700 training steps, the loss is 0.0111561, the validation accuracy is 0.9842\n",
      "after 37800 training steps, the loss is 0.0129434, the validation accuracy is 0.986\n",
      "after 37900 training steps, the loss is 0.0130457, the validation accuracy is 0.9844\n",
      "after 38000 training steps, the loss is 0.0127208, the validation accuracy is 0.9862\n",
      "after 38100 training steps, the loss is 0.0152566, the validation accuracy is 0.9822\n",
      "after 38200 training steps, the loss is 0.013629, the validation accuracy is 0.9852\n",
      "after 38300 training steps, the loss is 0.0169813, the validation accuracy is 0.9858\n",
      "after 38400 training steps, the loss is 0.00947986, the validation accuracy is 0.9856\n",
      "after 38500 training steps, the loss is 0.0100229, the validation accuracy is 0.9848\n",
      "after 38600 training steps, the loss is 0.015325, the validation accuracy is 0.984\n",
      "after 38700 training steps, the loss is 0.011091, the validation accuracy is 0.984\n",
      "after 38800 training steps, the loss is 0.0167238, the validation accuracy is 0.984\n",
      "after 38900 training steps, the loss is 0.0101053, the validation accuracy is 0.9836\n",
      "after 39000 training steps, the loss is 0.0146337, the validation accuracy is 0.9838\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 39100 training steps, the loss is 0.0132372, the validation accuracy is 0.985\n",
      "after 39200 training steps, the loss is 0.807396, the validation accuracy is 0.8542\n",
      "after 39300 training steps, the loss is 0.292378, the validation accuracy is 0.9638\n",
      "after 39400 training steps, the loss is 0.194922, the validation accuracy is 0.974\n",
      "after 39500 training steps, the loss is 0.111012, the validation accuracy is 0.9776\n",
      "after 39600 training steps, the loss is 0.0883577, the validation accuracy is 0.9802\n",
      "after 39700 training steps, the loss is 0.0929702, the validation accuracy is 0.9748\n",
      "after 39800 training steps, the loss is 0.0783626, the validation accuracy is 0.9792\n",
      "after 39900 training steps, the loss is 0.0479801, the validation accuracy is 0.9826\n",
      "after 40000 training steps, the loss is 0.0608435, the validation accuracy is 0.9798\n",
      "after 40100 training steps, the loss is 0.0328871, the validation accuracy is 0.9822\n",
      "after 40200 training steps, the loss is 0.0968453, the validation accuracy is 0.9822\n",
      "after 40300 training steps, the loss is 0.0319988, the validation accuracy is 0.9844\n",
      "after 40400 training steps, the loss is 0.0240299, the validation accuracy is 0.9826\n",
      "after 40500 training steps, the loss is 0.0264633, the validation accuracy is 0.9804\n",
      "after 40600 training steps, the loss is 0.0412231, the validation accuracy is 0.9824\n",
      "after 40700 training steps, the loss is 0.122675, the validation accuracy is 0.9696\n",
      "after 40800 training steps, the loss is 0.0485327, the validation accuracy is 0.9826\n",
      "after 40900 training steps, the loss is 0.0234563, the validation accuracy is 0.9844\n",
      "after 41000 training steps, the loss is 0.0235259, the validation accuracy is 0.9838\n",
      "after 41100 training steps, the loss is 0.028279, the validation accuracy is 0.9844\n",
      "after 41200 training steps, the loss is 0.0264799, the validation accuracy is 0.9838\n",
      "after 41300 training steps, the loss is 0.0284768, the validation accuracy is 0.9832\n",
      "after 41400 training steps, the loss is 0.0518826, the validation accuracy is 0.982\n",
      "after 41500 training steps, the loss is 0.0278421, the validation accuracy is 0.9836\n",
      "after 41600 training steps, the loss is 0.018237, the validation accuracy is 0.9834\n",
      "after 41700 training steps, the loss is 0.016829, the validation accuracy is 0.9846\n",
      "after 41800 training steps, the loss is 0.0313985, the validation accuracy is 0.9826\n",
      "after 41900 training steps, the loss is 0.0295452, the validation accuracy is 0.984\n",
      "after 42000 training steps, the loss is 0.0237004, the validation accuracy is 0.9832\n",
      "after 42100 training steps, the loss is 0.025299, the validation accuracy is 0.984\n",
      "after 42200 training steps, the loss is 0.0152724, the validation accuracy is 0.9846\n",
      "after 42300 training steps, the loss is 0.0307733, the validation accuracy is 0.9816\n",
      "after 42400 training steps, the loss is 0.0174797, the validation accuracy is 0.9832\n",
      "after 42500 training steps, the loss is 0.0195366, the validation accuracy is 0.9832\n",
      "after 42600 training steps, the loss is 0.0285734, the validation accuracy is 0.9832\n",
      "after 42700 training steps, the loss is 0.0155584, the validation accuracy is 0.9848\n",
      "after 42800 training steps, the loss is 0.0159245, the validation accuracy is 0.9844\n",
      "after 42900 training steps, the loss is 0.0160469, the validation accuracy is 0.9844\n",
      "after 43000 training steps, the loss is 0.021854, the validation accuracy is 0.9852\n",
      "after 43100 training steps, the loss is 0.0228767, the validation accuracy is 0.9832\n",
      "after 43200 training steps, the loss is 0.0160099, the validation accuracy is 0.9824\n",
      "after 43300 training steps, the loss is 0.0191171, the validation accuracy is 0.9848\n",
      "after 43400 training steps, the loss is 0.0287026, the validation accuracy is 0.9842\n",
      "after 43500 training steps, the loss is 0.0171196, the validation accuracy is 0.984\n",
      "after 43600 training steps, the loss is 0.018059, the validation accuracy is 0.984\n",
      "after 43700 training steps, the loss is 0.0173177, the validation accuracy is 0.983\n",
      "after 43800 training steps, the loss is 0.0264442, the validation accuracy is 0.9838\n",
      "after 43900 training steps, the loss is 0.0314389, the validation accuracy is 0.9846\n",
      "after 44000 training steps, the loss is 0.013686, the validation accuracy is 0.984\n",
      "after 44100 training steps, the loss is 0.0381985, the validation accuracy is 0.9828\n",
      "after 44200 training steps, the loss is 0.0246251, the validation accuracy is 0.9848\n",
      "after 44300 training steps, the loss is 0.0182989, the validation accuracy is 0.9848\n",
      "after 44400 training steps, the loss is 0.015874, the validation accuracy is 0.9852\n",
      "after 44500 training steps, the loss is 0.0159613, the validation accuracy is 0.9842\n",
      "after 44600 training steps, the loss is 0.0179538, the validation accuracy is 0.984\n",
      "after 44700 training steps, the loss is 0.0154667, the validation accuracy is 0.9842\n",
      "after 44800 training steps, the loss is 0.0166201, the validation accuracy is 0.9836\n",
      "after 44900 training steps, the loss is 0.0191552, the validation accuracy is 0.9844\n",
      "after 45000 training steps, the loss is 0.0235187, the validation accuracy is 0.9842\n",
      "after 45100 training steps, the loss is 0.0136478, the validation accuracy is 0.985\n",
      "after 45200 training steps, the loss is 0.0115897, the validation accuracy is 0.9854\n",
      "after 45300 training steps, the loss is 0.0166222, the validation accuracy is 0.9846\n",
      "after 45400 training steps, the loss is 0.0155791, the validation accuracy is 0.984\n",
      "after 45500 training steps, the loss is 0.0124455, the validation accuracy is 0.9838\n",
      "after 45600 training steps, the loss is 0.0142177, the validation accuracy is 0.9834\n",
      "after 45700 training steps, the loss is 0.0294685, the validation accuracy is 0.9852\n",
      "after 45800 training steps, the loss is 0.0162657, the validation accuracy is 0.9848\n",
      "after 45900 training steps, the loss is 0.0285812, the validation accuracy is 0.9848\n",
      "after 46000 training steps, the loss is 0.018603, the validation accuracy is 0.9848\n",
      "after 46100 training steps, the loss is 0.0175145, the validation accuracy is 0.9832\n",
      "after 46200 training steps, the loss is 0.0137694, the validation accuracy is 0.9848\n",
      "after 46300 training steps, the loss is 0.0136887, the validation accuracy is 0.9866\n",
      "after 46400 training steps, the loss is 0.0123387, the validation accuracy is 0.9846\n",
      "after 46500 training steps, the loss is 0.0130141, the validation accuracy is 0.985\n",
      "after 46600 training steps, the loss is 0.0137596, the validation accuracy is 0.9856\n",
      "after 46700 training steps, the loss is 0.0178241, the validation accuracy is 0.984\n",
      "after 46800 training steps, the loss is 0.0119773, the validation accuracy is 0.9844\n",
      "after 46900 training steps, the loss is 0.0195016, the validation accuracy is 0.9852\n",
      "after 47000 training steps, the loss is 0.0132555, the validation accuracy is 0.986\n",
      "after 47100 training steps, the loss is 0.0116074, the validation accuracy is 0.9842\n",
      "after 47200 training steps, the loss is 0.0124553, the validation accuracy is 0.9842\n",
      "after 47300 training steps, the loss is 0.0149796, the validation accuracy is 0.983\n",
      "after 47400 training steps, the loss is 0.0120987, the validation accuracy is 0.985\n",
      "after 47500 training steps, the loss is 0.011591, the validation accuracy is 0.9846\n",
      "after 47600 training steps, the loss is 0.020594, the validation accuracy is 0.985\n",
      "after 47700 training steps, the loss is 0.012695, the validation accuracy is 0.986\n",
      "after 47800 training steps, the loss is 0.011964, the validation accuracy is 0.985\n",
      "after 47900 training steps, the loss is 0.0166, the validation accuracy is 0.9844\n",
      "after 48000 training steps, the loss is 0.0138018, the validation accuracy is 0.985\n",
      "after 48100 training steps, the loss is 0.0120788, the validation accuracy is 0.9838\n",
      "after 48200 training steps, the loss is 0.0102143, the validation accuracy is 0.9832\n",
      "after 48300 training steps, the loss is 0.011437, the validation accuracy is 0.985\n",
      "after 48400 training steps, the loss is 0.0191578, the validation accuracy is 0.9842\n",
      "after 48500 training steps, the loss is 0.0115097, the validation accuracy is 0.9848\n",
      "after 48600 training steps, the loss is 0.0161769, the validation accuracy is 0.985\n",
      "after 48700 training steps, the loss is 0.0440303, the validation accuracy is 0.9744\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 48800 training steps, the loss is 0.0400482, the validation accuracy is 0.982\n",
      "after 48900 training steps, the loss is 0.0277698, the validation accuracy is 0.9852\n",
      "after 49000 training steps, the loss is 0.0152438, the validation accuracy is 0.985\n",
      "after 49100 training steps, the loss is 0.0143098, the validation accuracy is 0.9846\n",
      "after 49200 training steps, the loss is 0.0146016, the validation accuracy is 0.9854\n",
      "after 49300 training steps, the loss is 0.0163279, the validation accuracy is 0.9836\n",
      "after 49400 training steps, the loss is 0.0134274, the validation accuracy is 0.9828\n",
      "after 49500 training steps, the loss is 0.0127733, the validation accuracy is 0.9846\n",
      "after 49600 training steps, the loss is 0.0174316, the validation accuracy is 0.9862\n",
      "after 49700 training steps, the loss is 0.0131689, the validation accuracy is 0.9858\n",
      "after 49800 training steps, the loss is 0.0178391, the validation accuracy is 0.9828\n",
      "after 49900 training steps, the loss is 0.0239808, the validation accuracy is 0.9848\n",
      "after 50000 training steps, the loss is 0.0174454, the validation accuracy is 0.9848\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.99987274\n",
      "the test accuarcy is: 0.983\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=64, training_step=50000, lr=0.3, lambda_flag='l2', lambda_value=0.01, initial_way=MSRA, \n",
    "      hidden1=1000, hidden2=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "提升了0.001，增加迭代次数这种暴力法好像没太大用还耗时，下面再增加一个隐层试试～"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 2.2644, the validation accuracy is 0.7682\n",
      "after 200 training steps, the loss is 1.65865, the validation accuracy is 0.8542\n",
      "after 300 training steps, the loss is 1.60252, the validation accuracy is 0.8836\n",
      "after 400 training steps, the loss is 1.16428, the validation accuracy is 0.9062\n",
      "after 500 training steps, the loss is 0.944019, the validation accuracy is 0.9232\n",
      "after 600 training steps, the loss is 1.10484, the validation accuracy is 0.9278\n",
      "after 700 training steps, the loss is 0.7906, the validation accuracy is 0.9408\n",
      "after 800 training steps, the loss is 0.520891, the validation accuracy is 0.9326\n",
      "after 900 training steps, the loss is 0.545908, the validation accuracy is 0.9502\n",
      "after 1000 training steps, the loss is 0.509463, the validation accuracy is 0.9372\n",
      "after 1100 training steps, the loss is 0.42715, the validation accuracy is 0.9496\n",
      "after 1200 training steps, the loss is 0.484429, the validation accuracy is 0.944\n",
      "after 1300 training steps, the loss is 0.308632, the validation accuracy is 0.953\n",
      "after 1400 training steps, the loss is 0.276916, the validation accuracy is 0.9574\n",
      "after 1500 training steps, the loss is 0.296742, the validation accuracy is 0.959\n",
      "after 1600 training steps, the loss is 0.327313, the validation accuracy is 0.958\n",
      "after 1700 training steps, the loss is 0.170751, the validation accuracy is 0.9572\n",
      "after 1800 training steps, the loss is 0.252982, the validation accuracy is 0.9596\n",
      "after 1900 training steps, the loss is 0.232738, the validation accuracy is 0.959\n",
      "after 2000 training steps, the loss is 0.0964115, the validation accuracy is 0.9652\n",
      "after 2100 training steps, the loss is 0.165199, the validation accuracy is 0.9582\n",
      "after 2200 training steps, the loss is 0.10112, the validation accuracy is 0.9658\n",
      "after 2300 training steps, the loss is 0.117974, the validation accuracy is 0.9624\n",
      "after 2400 training steps, the loss is 0.102299, the validation accuracy is 0.9662\n",
      "after 2500 training steps, the loss is 0.18668, the validation accuracy is 0.9678\n",
      "after 2600 training steps, the loss is 0.144042, the validation accuracy is 0.969\n",
      "after 2700 training steps, the loss is 0.071719, the validation accuracy is 0.9676\n",
      "after 2800 training steps, the loss is 0.0637723, the validation accuracy is 0.9708\n",
      "after 2900 training steps, the loss is 0.114122, the validation accuracy is 0.9686\n",
      "after 3000 training steps, the loss is 0.0680326, the validation accuracy is 0.9682\n",
      "after 3100 training steps, the loss is 0.0709522, the validation accuracy is 0.9644\n",
      "after 3200 training steps, the loss is 0.133917, the validation accuracy is 0.971\n",
      "after 3300 training steps, the loss is 0.0731131, the validation accuracy is 0.9654\n",
      "after 3400 training steps, the loss is 0.0667488, the validation accuracy is 0.9718\n",
      "after 3500 training steps, the loss is 0.163712, the validation accuracy is 0.9644\n",
      "after 3600 training steps, the loss is 0.0476277, the validation accuracy is 0.9728\n",
      "after 3700 training steps, the loss is 0.0692567, the validation accuracy is 0.973\n",
      "after 3800 training steps, the loss is 0.154242, the validation accuracy is 0.9704\n",
      "after 3900 training steps, the loss is 0.067146, the validation accuracy is 0.971\n",
      "after 4000 training steps, the loss is 0.292574, the validation accuracy is 0.9746\n",
      "after 4100 training steps, the loss is 0.07668, the validation accuracy is 0.9748\n",
      "after 4200 training steps, the loss is 0.0596328, the validation accuracy is 0.9746\n",
      "after 4300 training steps, the loss is 0.0495812, the validation accuracy is 0.975\n",
      "after 4400 training steps, the loss is 0.0611439, the validation accuracy is 0.975\n",
      "after 4500 training steps, the loss is 0.17825, the validation accuracy is 0.97\n",
      "after 4600 training steps, the loss is 0.0997125, the validation accuracy is 0.9714\n",
      "after 4700 training steps, the loss is 0.0679108, the validation accuracy is 0.9736\n",
      "after 4800 training steps, the loss is 0.0529556, the validation accuracy is 0.9744\n",
      "after 4900 training steps, the loss is 0.0394712, the validation accuracy is 0.974\n",
      "after 5000 training steps, the loss is 0.115186, the validation accuracy is 0.9776\n",
      "after 5100 training steps, the loss is 0.0422222, the validation accuracy is 0.9772\n",
      "after 5200 training steps, the loss is 0.069873, the validation accuracy is 0.9746\n",
      "after 5300 training steps, the loss is 0.0499183, the validation accuracy is 0.9754\n",
      "after 5400 training steps, the loss is 0.160757, the validation accuracy is 0.9738\n",
      "after 5500 training steps, the loss is 0.0970659, the validation accuracy is 0.9748\n",
      "after 5600 training steps, the loss is 0.062134, the validation accuracy is 0.9754\n",
      "after 5700 training steps, the loss is 0.0582668, the validation accuracy is 0.976\n",
      "after 5800 training steps, the loss is 0.0792576, the validation accuracy is 0.976\n",
      "after 5900 training steps, the loss is 0.0382206, the validation accuracy is 0.976\n",
      "after 6000 training steps, the loss is 0.0511926, the validation accuracy is 0.9768\n",
      "after 6100 training steps, the loss is 0.0947843, the validation accuracy is 0.9726\n",
      "after 6200 training steps, the loss is 0.356756, the validation accuracy is 0.9758\n",
      "after 6300 training steps, the loss is 0.110179, the validation accuracy is 0.9732\n",
      "after 6400 training steps, the loss is 0.0784854, the validation accuracy is 0.976\n",
      "after 6500 training steps, the loss is 0.0518893, the validation accuracy is 0.9786\n",
      "after 6600 training steps, the loss is 0.140475, the validation accuracy is 0.9758\n",
      "after 6700 training steps, the loss is 0.0701628, the validation accuracy is 0.9762\n",
      "after 6800 training steps, the loss is 0.0735117, the validation accuracy is 0.9776\n",
      "after 6900 training steps, the loss is 0.0847194, the validation accuracy is 0.9738\n",
      "after 7000 training steps, the loss is 0.102168, the validation accuracy is 0.9746\n",
      "after 7100 training steps, the loss is 0.0594379, the validation accuracy is 0.981\n",
      "after 7200 training steps, the loss is 0.103193, the validation accuracy is 0.9756\n",
      "after 7300 training steps, the loss is 0.0357793, the validation accuracy is 0.9802\n",
      "after 7400 training steps, the loss is 0.0330088, the validation accuracy is 0.9778\n",
      "after 7500 training steps, the loss is 0.0581319, the validation accuracy is 0.9754\n",
      "after 7600 training steps, the loss is 0.0389179, the validation accuracy is 0.9802\n",
      "after 7700 training steps, the loss is 0.0748995, the validation accuracy is 0.981\n",
      "after 7800 training steps, the loss is 0.0260552, the validation accuracy is 0.9798\n",
      "after 7900 training steps, the loss is 0.116901, the validation accuracy is 0.977\n",
      "after 8000 training steps, the loss is 0.0504014, the validation accuracy is 0.9798\n",
      "after 8100 training steps, the loss is 0.0409248, the validation accuracy is 0.9778\n",
      "after 8200 training steps, the loss is 0.0378845, the validation accuracy is 0.978\n",
      "after 8300 training steps, the loss is 0.030943, the validation accuracy is 0.9794\n",
      "after 8400 training steps, the loss is 0.0912892, the validation accuracy is 0.9784\n",
      "after 8500 training steps, the loss is 0.0894868, the validation accuracy is 0.9738\n",
      "after 8600 training steps, the loss is 0.0559291, the validation accuracy is 0.9802\n",
      "after 8700 training steps, the loss is 0.0493675, the validation accuracy is 0.9806\n",
      "after 8800 training steps, the loss is 0.143004, the validation accuracy is 0.9808\n",
      "after 8900 training steps, the loss is 0.0332043, the validation accuracy is 0.9792\n",
      "after 9000 training steps, the loss is 0.0427793, the validation accuracy is 0.9806\n",
      "after 9100 training steps, the loss is 0.0618413, the validation accuracy is 0.9814\n",
      "after 9200 training steps, the loss is 0.0340943, the validation accuracy is 0.9814\n",
      "after 9300 training steps, the loss is 0.0616907, the validation accuracy is 0.9814\n",
      "after 9400 training steps, the loss is 0.0600344, the validation accuracy is 0.9774\n",
      "after 9500 training steps, the loss is 0.0480736, the validation accuracy is 0.9786\n",
      "after 9600 training steps, the loss is 0.0810321, the validation accuracy is 0.9808\n",
      "after 9700 training steps, the loss is 0.0359848, the validation accuracy is 0.9798\n",
      "after 9800 training steps, the loss is 0.079879, the validation accuracy is 0.9808\n",
      "after 9900 training steps, the loss is 0.0334947, the validation accuracy is 0.9822\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10000 training steps, the loss is 0.0319031, the validation accuracy is 0.9816\n",
      "after 10100 training steps, the loss is 0.0443662, the validation accuracy is 0.9808\n",
      "after 10200 training steps, the loss is 0.0320431, the validation accuracy is 0.9824\n",
      "after 10300 training steps, the loss is 0.0435988, the validation accuracy is 0.9818\n",
      "after 10400 training steps, the loss is 0.0495222, the validation accuracy is 0.9816\n",
      "after 10500 training steps, the loss is 0.0216315, the validation accuracy is 0.9802\n",
      "after 10600 training steps, the loss is 0.0461716, the validation accuracy is 0.976\n",
      "after 10700 training steps, the loss is 0.020968, the validation accuracy is 0.9778\n",
      "after 10800 training steps, the loss is 0.0301632, the validation accuracy is 0.9814\n",
      "after 10900 training steps, the loss is 0.0283998, the validation accuracy is 0.9814\n",
      "after 11000 training steps, the loss is 0.0440236, the validation accuracy is 0.98\n",
      "after 11100 training steps, the loss is 0.0384909, the validation accuracy is 0.9812\n",
      "after 11200 training steps, the loss is 0.0481736, the validation accuracy is 0.9806\n",
      "after 11300 training steps, the loss is 0.0262979, the validation accuracy is 0.9814\n",
      "after 11400 training steps, the loss is 0.0529486, the validation accuracy is 0.9806\n",
      "after 11500 training steps, the loss is 0.0472527, the validation accuracy is 0.9812\n",
      "after 11600 training steps, the loss is 0.0343243, the validation accuracy is 0.9814\n",
      "after 11700 training steps, the loss is 0.0423648, the validation accuracy is 0.9808\n",
      "after 11800 training steps, the loss is 0.0632151, the validation accuracy is 0.9818\n",
      "after 11900 training steps, the loss is 0.0516348, the validation accuracy is 0.9814\n",
      "after 12000 training steps, the loss is 0.0356812, the validation accuracy is 0.9802\n",
      "after 12100 training steps, the loss is 0.0302421, the validation accuracy is 0.9818\n",
      "after 12200 training steps, the loss is 0.0244879, the validation accuracy is 0.9802\n",
      "after 12300 training steps, the loss is 0.0494604, the validation accuracy is 0.9818\n",
      "after 12400 training steps, the loss is 0.0444066, the validation accuracy is 0.9812\n",
      "after 12500 training steps, the loss is 0.0275104, the validation accuracy is 0.9812\n",
      "after 12600 training steps, the loss is 0.0687198, the validation accuracy is 0.981\n",
      "after 12700 training steps, the loss is 0.0265468, the validation accuracy is 0.9814\n",
      "after 12800 training steps, the loss is 0.0237792, the validation accuracy is 0.9832\n",
      "after 12900 training steps, the loss is 0.0232191, the validation accuracy is 0.9804\n",
      "after 13000 training steps, the loss is 0.0302135, the validation accuracy is 0.982\n",
      "after 13100 training steps, the loss is 0.0216158, the validation accuracy is 0.9812\n",
      "after 13200 training steps, the loss is 0.0206998, the validation accuracy is 0.9818\n",
      "after 13300 training steps, the loss is 0.0270457, the validation accuracy is 0.9792\n",
      "after 13400 training steps, the loss is 0.0327024, the validation accuracy is 0.982\n",
      "after 13500 training steps, the loss is 0.0356406, the validation accuracy is 0.9802\n",
      "after 13600 training steps, the loss is 0.0495826, the validation accuracy is 0.981\n",
      "after 13700 training steps, the loss is 0.0274619, the validation accuracy is 0.9824\n",
      "after 13800 training steps, the loss is 0.0241906, the validation accuracy is 0.982\n",
      "after 13900 training steps, the loss is 0.0297561, the validation accuracy is 0.9802\n",
      "after 14000 training steps, the loss is 0.0433269, the validation accuracy is 0.9792\n",
      "after 14100 training steps, the loss is 0.0227542, the validation accuracy is 0.9818\n",
      "after 14200 training steps, the loss is 0.142682, the validation accuracy is 0.9786\n",
      "after 14300 training steps, the loss is 0.0246346, the validation accuracy is 0.9812\n",
      "after 14400 training steps, the loss is 0.0245474, the validation accuracy is 0.9812\n",
      "after 14500 training steps, the loss is 0.0243636, the validation accuracy is 0.9842\n",
      "after 14600 training steps, the loss is 0.0260161, the validation accuracy is 0.981\n",
      "after 14700 training steps, the loss is 0.0195244, the validation accuracy is 0.9792\n",
      "after 14800 training steps, the loss is 0.0333251, the validation accuracy is 0.9818\n",
      "after 14900 training steps, the loss is 0.0247975, the validation accuracy is 0.9822\n",
      "after 15000 training steps, the loss is 0.0207742, the validation accuracy is 0.9826\n",
      "after 15100 training steps, the loss is 0.0175331, the validation accuracy is 0.9808\n",
      "after 15200 training steps, the loss is 0.0283284, the validation accuracy is 0.9824\n",
      "after 15300 training steps, the loss is 0.0238983, the validation accuracy is 0.9828\n",
      "after 15400 training steps, the loss is 0.0177049, the validation accuracy is 0.9806\n",
      "after 15500 training steps, the loss is 0.0215144, the validation accuracy is 0.9806\n",
      "after 15600 training steps, the loss is 0.0423845, the validation accuracy is 0.9814\n",
      "after 15700 training steps, the loss is 0.0184411, the validation accuracy is 0.9814\n",
      "after 15800 training steps, the loss is 0.0252766, the validation accuracy is 0.9814\n",
      "after 15900 training steps, the loss is 0.0375908, the validation accuracy is 0.9816\n",
      "after 16000 training steps, the loss is 0.0167202, the validation accuracy is 0.9818\n",
      "after 16100 training steps, the loss is 0.0404413, the validation accuracy is 0.9812\n",
      "after 16200 training steps, the loss is 0.0575435, the validation accuracy is 0.9808\n",
      "after 16300 training steps, the loss is 0.0202547, the validation accuracy is 0.9808\n",
      "after 16400 training steps, the loss is 0.023038, the validation accuracy is 0.9818\n",
      "after 16500 training steps, the loss is 0.0540834, the validation accuracy is 0.9796\n",
      "after 16600 training steps, the loss is 0.0265647, the validation accuracy is 0.9832\n",
      "after 16700 training steps, the loss is 0.0753369, the validation accuracy is 0.9796\n",
      "after 16800 training steps, the loss is 0.0327503, the validation accuracy is 0.9812\n",
      "after 16900 training steps, the loss is 0.0140449, the validation accuracy is 0.9828\n",
      "after 17000 training steps, the loss is 0.024292, the validation accuracy is 0.9818\n",
      "after 17100 training steps, the loss is 0.0247446, the validation accuracy is 0.9814\n",
      "after 17200 training steps, the loss is 0.0180281, the validation accuracy is 0.982\n",
      "after 17300 training steps, the loss is 0.0191822, the validation accuracy is 0.982\n",
      "after 17400 training steps, the loss is 0.0384419, the validation accuracy is 0.9836\n",
      "after 17500 training steps, the loss is 0.0597008, the validation accuracy is 0.9814\n",
      "after 17600 training steps, the loss is 0.0154775, the validation accuracy is 0.9832\n",
      "after 17700 training steps, the loss is 0.014227, the validation accuracy is 0.9822\n",
      "after 17800 training steps, the loss is 0.0602141, the validation accuracy is 0.9824\n",
      "after 17900 training steps, the loss is 0.0319118, the validation accuracy is 0.9806\n",
      "after 18000 training steps, the loss is 0.0170554, the validation accuracy is 0.9812\n",
      "after 18100 training steps, the loss is 0.0142312, the validation accuracy is 0.9818\n",
      "after 18200 training steps, the loss is 0.0141706, the validation accuracy is 0.9818\n",
      "after 18300 training steps, the loss is 0.0163374, the validation accuracy is 0.9824\n",
      "after 18400 training steps, the loss is 0.0221432, the validation accuracy is 0.982\n",
      "after 18500 training steps, the loss is 0.021266, the validation accuracy is 0.9776\n",
      "after 18600 training steps, the loss is 0.0150116, the validation accuracy is 0.9814\n",
      "after 18700 training steps, the loss is 0.025245, the validation accuracy is 0.9816\n",
      "after 18800 training steps, the loss is 0.035514, the validation accuracy is 0.9802\n",
      "after 18900 training steps, the loss is 0.0214788, the validation accuracy is 0.9806\n",
      "after 19000 training steps, the loss is 0.0167209, the validation accuracy is 0.9804\n",
      "after 19100 training steps, the loss is 0.0156832, the validation accuracy is 0.9828\n",
      "after 19200 training steps, the loss is 0.0182783, the validation accuracy is 0.982\n",
      "after 19300 training steps, the loss is 0.0195167, the validation accuracy is 0.984\n",
      "after 19400 training steps, the loss is 0.0209595, the validation accuracy is 0.9828\n",
      "after 19500 training steps, the loss is 0.0158648, the validation accuracy is 0.9824\n",
      "after 19600 training steps, the loss is 0.0210024, the validation accuracy is 0.9828\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19700 training steps, the loss is 0.0138, the validation accuracy is 0.9822\n",
      "after 19800 training steps, the loss is 0.0196985, the validation accuracy is 0.9824\n",
      "after 19900 training steps, the loss is 0.0275593, the validation accuracy is 0.9828\n",
      "after 20000 training steps, the loss is 0.0160106, the validation accuracy is 0.9816\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.9991818\n",
      "the test accuarcy is: 0.9812\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=64, training_step=20000, lr=0.1, lambda_flag='l2', lambda_value=0.01, initial_way=MSRA, \n",
    "      hidden1=1000, hidden2=1000, hidden3=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "基本没啥变化诶～，再改变一下batch_size吧"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 4.5393, the validation accuracy is 0.6856\n",
      "after 200 training steps, the loss is 3.50715, the validation accuracy is 0.881\n",
      "after 300 training steps, the loss is 2.72296, the validation accuracy is 0.9226\n",
      "after 400 training steps, the loss is 2.27711, the validation accuracy is 0.9322\n",
      "after 500 training steps, the loss is 1.88716, the validation accuracy is 0.935\n",
      "after 600 training steps, the loss is 1.5375, the validation accuracy is 0.9412\n",
      "after 700 training steps, the loss is 1.28633, the validation accuracy is 0.9418\n",
      "after 800 training steps, the loss is 1.10762, the validation accuracy is 0.9404\n",
      "after 900 training steps, the loss is 0.96506, the validation accuracy is 0.9488\n",
      "after 1000 training steps, the loss is 0.748334, the validation accuracy is 0.9544\n",
      "after 1100 training steps, the loss is 0.732026, the validation accuracy is 0.9564\n",
      "after 1200 training steps, the loss is 0.692142, the validation accuracy is 0.9562\n",
      "after 1300 training steps, the loss is 0.510599, the validation accuracy is 0.9548\n",
      "after 1400 training steps, the loss is 0.477271, the validation accuracy is 0.9602\n",
      "after 1500 training steps, the loss is 0.378517, the validation accuracy is 0.9598\n",
      "after 1600 training steps, the loss is 0.286431, the validation accuracy is 0.9606\n",
      "after 1700 training steps, the loss is 0.33302, the validation accuracy is 0.9612\n",
      "after 1800 training steps, the loss is 0.216841, the validation accuracy is 0.9656\n",
      "after 1900 training steps, the loss is 0.26412, the validation accuracy is 0.9658\n",
      "after 2000 training steps, the loss is 0.230171, the validation accuracy is 0.9662\n",
      "after 2100 training steps, the loss is 0.197213, the validation accuracy is 0.9668\n",
      "after 2200 training steps, the loss is 0.217974, the validation accuracy is 0.9684\n",
      "after 2300 training steps, the loss is 0.176523, the validation accuracy is 0.9706\n",
      "after 2400 training steps, the loss is 0.265105, the validation accuracy is 0.9692\n",
      "after 2500 training steps, the loss is 0.198163, the validation accuracy is 0.9708\n",
      "after 2600 training steps, the loss is 0.17922, the validation accuracy is 0.9712\n",
      "after 2700 training steps, the loss is 0.104446, the validation accuracy is 0.9728\n",
      "after 2800 training steps, the loss is 0.0841106, the validation accuracy is 0.974\n",
      "after 2900 training steps, the loss is 0.143127, the validation accuracy is 0.9702\n",
      "after 3000 training steps, the loss is 0.0830937, the validation accuracy is 0.9732\n",
      "after 3100 training steps, the loss is 0.0879205, the validation accuracy is 0.9748\n",
      "after 3200 training steps, the loss is 0.101259, the validation accuracy is 0.973\n",
      "after 3300 training steps, the loss is 0.0952952, the validation accuracy is 0.9734\n",
      "after 3400 training steps, the loss is 0.170119, the validation accuracy is 0.9746\n",
      "after 3500 training steps, the loss is 0.0583517, the validation accuracy is 0.971\n",
      "after 3600 training steps, the loss is 0.0860235, the validation accuracy is 0.976\n",
      "after 3700 training steps, the loss is 0.185503, the validation accuracy is 0.9776\n",
      "after 3800 training steps, the loss is 0.0864248, the validation accuracy is 0.9744\n",
      "after 3900 training steps, the loss is 0.0787318, the validation accuracy is 0.9756\n",
      "after 4000 training steps, the loss is 0.0748002, the validation accuracy is 0.975\n",
      "after 4100 training steps, the loss is 0.0920119, the validation accuracy is 0.9746\n",
      "after 4200 training steps, the loss is 0.107817, the validation accuracy is 0.9766\n",
      "after 4300 training steps, the loss is 0.0997943, the validation accuracy is 0.9774\n",
      "after 4400 training steps, the loss is 0.107526, the validation accuracy is 0.977\n",
      "after 4500 training steps, the loss is 0.117127, the validation accuracy is 0.9744\n",
      "after 4600 training steps, the loss is 0.0523934, the validation accuracy is 0.9806\n",
      "after 4700 training steps, the loss is 0.0616984, the validation accuracy is 0.9746\n",
      "after 4800 training steps, the loss is 0.0612813, the validation accuracy is 0.975\n",
      "after 4900 training steps, the loss is 0.0634226, the validation accuracy is 0.978\n",
      "after 5000 training steps, the loss is 0.054386, the validation accuracy is 0.9788\n",
      "after 5100 training steps, the loss is 0.0447033, the validation accuracy is 0.9792\n",
      "after 5200 training steps, the loss is 0.04612, the validation accuracy is 0.9784\n",
      "after 5300 training steps, the loss is 0.0964233, the validation accuracy is 0.9738\n",
      "after 5400 training steps, the loss is 0.07795, the validation accuracy is 0.9798\n",
      "after 5500 training steps, the loss is 0.0337577, the validation accuracy is 0.977\n",
      "after 5600 training steps, the loss is 0.0744012, the validation accuracy is 0.9776\n",
      "after 5700 training steps, the loss is 0.0625027, the validation accuracy is 0.9772\n",
      "after 5800 training steps, the loss is 0.0989543, the validation accuracy is 0.9774\n",
      "after 5900 training steps, the loss is 0.0631819, the validation accuracy is 0.9806\n",
      "after 6000 training steps, the loss is 0.0439084, the validation accuracy is 0.9786\n",
      "after 6100 training steps, the loss is 0.0416827, the validation accuracy is 0.9786\n",
      "after 6200 training steps, the loss is 0.120464, the validation accuracy is 0.9794\n",
      "after 6300 training steps, the loss is 0.060797, the validation accuracy is 0.978\n",
      "after 6400 training steps, the loss is 0.0373506, the validation accuracy is 0.9796\n",
      "after 6500 training steps, the loss is 0.0539689, the validation accuracy is 0.9794\n",
      "after 6600 training steps, the loss is 0.0536797, the validation accuracy is 0.979\n",
      "after 6700 training steps, the loss is 0.0770285, the validation accuracy is 0.9778\n",
      "after 6800 training steps, the loss is 0.0625089, the validation accuracy is 0.9778\n",
      "after 6900 training steps, the loss is 0.037042, the validation accuracy is 0.9812\n",
      "after 7000 training steps, the loss is 0.0413225, the validation accuracy is 0.9794\n",
      "after 7100 training steps, the loss is 0.0263639, the validation accuracy is 0.9812\n",
      "after 7200 training steps, the loss is 0.0435207, the validation accuracy is 0.9732\n",
      "after 7300 training steps, the loss is 0.0343286, the validation accuracy is 0.9814\n",
      "after 7400 training steps, the loss is 0.0454579, the validation accuracy is 0.9816\n",
      "after 7500 training steps, the loss is 0.0450697, the validation accuracy is 0.979\n",
      "after 7600 training steps, the loss is 0.040399, the validation accuracy is 0.9806\n",
      "after 7700 training steps, the loss is 0.033872, the validation accuracy is 0.9788\n",
      "after 7800 training steps, the loss is 0.0370797, the validation accuracy is 0.9802\n",
      "after 7900 training steps, the loss is 0.0411121, the validation accuracy is 0.9812\n",
      "after 8000 training steps, the loss is 0.043628, the validation accuracy is 0.9824\n",
      "after 8100 training steps, the loss is 0.0427059, the validation accuracy is 0.9832\n",
      "after 8200 training steps, the loss is 0.0308407, the validation accuracy is 0.9818\n",
      "after 8300 training steps, the loss is 0.0601869, the validation accuracy is 0.9782\n",
      "after 8400 training steps, the loss is 0.0365559, the validation accuracy is 0.9824\n",
      "after 8500 training steps, the loss is 0.0334679, the validation accuracy is 0.9818\n",
      "after 8600 training steps, the loss is 0.0406966, the validation accuracy is 0.981\n",
      "after 8700 training steps, the loss is 0.0407285, the validation accuracy is 0.9822\n",
      "after 8800 training steps, the loss is 0.0362339, the validation accuracy is 0.9818\n",
      "after 8900 training steps, the loss is 0.0414424, the validation accuracy is 0.9808\n",
      "after 9000 training steps, the loss is 0.0633336, the validation accuracy is 0.9802\n",
      "after 9100 training steps, the loss is 0.0308081, the validation accuracy is 0.9824\n",
      "after 9200 training steps, the loss is 0.0206897, the validation accuracy is 0.9824\n",
      "after 9300 training steps, the loss is 0.0417115, the validation accuracy is 0.9818\n",
      "after 9400 training steps, the loss is 0.0271444, the validation accuracy is 0.9822\n",
      "after 9500 training steps, the loss is 0.0245675, the validation accuracy is 0.9808\n",
      "after 9600 training steps, the loss is 0.0241926, the validation accuracy is 0.9806\n",
      "after 9700 training steps, the loss is 0.0253978, the validation accuracy is 0.9818\n",
      "after 9800 training steps, the loss is 0.0553437, the validation accuracy is 0.9828\n",
      "after 9900 training steps, the loss is 0.0279565, the validation accuracy is 0.9802\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 10000 training steps, the loss is 0.0231, the validation accuracy is 0.9828\n",
      "after 10100 training steps, the loss is 0.0314756, the validation accuracy is 0.9812\n",
      "after 10200 training steps, the loss is 0.030366, the validation accuracy is 0.9812\n",
      "after 10300 training steps, the loss is 0.0498295, the validation accuracy is 0.982\n",
      "after 10400 training steps, the loss is 0.0207276, the validation accuracy is 0.9814\n",
      "after 10500 training steps, the loss is 0.0298364, the validation accuracy is 0.9824\n",
      "after 10600 training steps, the loss is 0.0332687, the validation accuracy is 0.9814\n",
      "after 10700 training steps, the loss is 0.037642, the validation accuracy is 0.9822\n",
      "after 10800 training steps, the loss is 0.0261808, the validation accuracy is 0.9808\n",
      "after 10900 training steps, the loss is 0.0304246, the validation accuracy is 0.982\n",
      "after 11000 training steps, the loss is 0.0218128, the validation accuracy is 0.983\n",
      "after 11100 training steps, the loss is 0.0427751, the validation accuracy is 0.9826\n",
      "after 11200 training steps, the loss is 0.0336984, the validation accuracy is 0.9814\n",
      "after 11300 training steps, the loss is 0.0276465, the validation accuracy is 0.982\n",
      "after 11400 training steps, the loss is 0.029163, the validation accuracy is 0.9826\n",
      "after 11500 training steps, the loss is 0.0227735, the validation accuracy is 0.9828\n",
      "after 11600 training steps, the loss is 0.0239794, the validation accuracy is 0.9824\n",
      "after 11700 training steps, the loss is 0.0229836, the validation accuracy is 0.9822\n",
      "after 11800 training steps, the loss is 0.0777631, the validation accuracy is 0.979\n",
      "after 11900 training steps, the loss is 0.0230143, the validation accuracy is 0.9822\n",
      "after 12000 training steps, the loss is 0.0212183, the validation accuracy is 0.9802\n",
      "after 12100 training steps, the loss is 0.0207982, the validation accuracy is 0.9826\n",
      "after 12200 training steps, the loss is 0.0223091, the validation accuracy is 0.982\n",
      "after 12300 training steps, the loss is 0.0329473, the validation accuracy is 0.9828\n",
      "after 12400 training steps, the loss is 0.0193487, the validation accuracy is 0.9822\n",
      "after 12500 training steps, the loss is 0.134283, the validation accuracy is 0.983\n",
      "after 12600 training steps, the loss is 0.0186753, the validation accuracy is 0.9808\n",
      "after 12700 training steps, the loss is 0.0162926, the validation accuracy is 0.9832\n",
      "after 12800 training steps, the loss is 0.0234107, the validation accuracy is 0.9824\n",
      "after 12900 training steps, the loss is 0.0331242, the validation accuracy is 0.9828\n",
      "after 13000 training steps, the loss is 0.0226102, the validation accuracy is 0.9842\n",
      "after 13100 training steps, the loss is 0.0231804, the validation accuracy is 0.9822\n",
      "after 13200 training steps, the loss is 0.0188106, the validation accuracy is 0.9828\n",
      "after 13300 training steps, the loss is 0.0187599, the validation accuracy is 0.9826\n",
      "after 13400 training steps, the loss is 0.0232093, the validation accuracy is 0.9836\n",
      "after 13500 training steps, the loss is 0.0256504, the validation accuracy is 0.9824\n",
      "after 13600 training steps, the loss is 0.025969, the validation accuracy is 0.9814\n",
      "after 13700 training steps, the loss is 0.0210942, the validation accuracy is 0.9838\n",
      "after 13800 training steps, the loss is 0.027054, the validation accuracy is 0.983\n",
      "after 13900 training steps, the loss is 0.0166603, the validation accuracy is 0.9826\n",
      "after 14000 training steps, the loss is 0.0154403, the validation accuracy is 0.9834\n",
      "after 14100 training steps, the loss is 0.0193814, the validation accuracy is 0.982\n",
      "after 14200 training steps, the loss is 0.0192293, the validation accuracy is 0.9808\n",
      "after 14300 training steps, the loss is 0.0211085, the validation accuracy is 0.983\n",
      "after 14400 training steps, the loss is 0.0186837, the validation accuracy is 0.9832\n",
      "after 14500 training steps, the loss is 0.0229055, the validation accuracy is 0.9816\n",
      "after 14600 training steps, the loss is 0.0245281, the validation accuracy is 0.9812\n",
      "after 14700 training steps, the loss is 0.0151743, the validation accuracy is 0.9814\n",
      "after 14800 training steps, the loss is 0.0243767, the validation accuracy is 0.9824\n",
      "after 14900 training steps, the loss is 0.0298212, the validation accuracy is 0.9814\n",
      "after 15000 training steps, the loss is 0.0211126, the validation accuracy is 0.9822\n",
      "after 15100 training steps, the loss is 0.0221127, the validation accuracy is 0.9814\n",
      "after 15200 training steps, the loss is 0.0155096, the validation accuracy is 0.984\n",
      "after 15300 training steps, the loss is 0.0179378, the validation accuracy is 0.9824\n",
      "after 15400 training steps, the loss is 0.0165976, the validation accuracy is 0.982\n",
      "after 15500 training steps, the loss is 0.0176941, the validation accuracy is 0.9824\n",
      "after 15600 training steps, the loss is 0.0189621, the validation accuracy is 0.9822\n",
      "after 15700 training steps, the loss is 0.0203396, the validation accuracy is 0.9828\n",
      "after 15800 training steps, the loss is 0.0214979, the validation accuracy is 0.9812\n",
      "after 15900 training steps, the loss is 0.022233, the validation accuracy is 0.9828\n",
      "after 16000 training steps, the loss is 0.0210863, the validation accuracy is 0.9826\n",
      "after 16100 training steps, the loss is 0.0173338, the validation accuracy is 0.982\n",
      "after 16200 training steps, the loss is 0.014776, the validation accuracy is 0.9838\n",
      "after 16300 training steps, the loss is 0.0154334, the validation accuracy is 0.9816\n",
      "after 16400 training steps, the loss is 0.0346661, the validation accuracy is 0.9834\n",
      "after 16500 training steps, the loss is 0.0192307, the validation accuracy is 0.9832\n",
      "after 16600 training steps, the loss is 0.0249742, the validation accuracy is 0.983\n",
      "after 16700 training steps, the loss is 0.0170597, the validation accuracy is 0.983\n",
      "after 16800 training steps, the loss is 0.0159445, the validation accuracy is 0.9836\n",
      "after 16900 training steps, the loss is 0.0185301, the validation accuracy is 0.983\n",
      "after 17000 training steps, the loss is 0.0178563, the validation accuracy is 0.983\n",
      "after 17100 training steps, the loss is 0.0249129, the validation accuracy is 0.9808\n",
      "after 17200 training steps, the loss is 0.0159797, the validation accuracy is 0.983\n",
      "after 17300 training steps, the loss is 0.0130391, the validation accuracy is 0.9828\n",
      "after 17400 training steps, the loss is 0.0144751, the validation accuracy is 0.983\n",
      "after 17500 training steps, the loss is 0.0184168, the validation accuracy is 0.982\n",
      "after 17600 training steps, the loss is 0.0144867, the validation accuracy is 0.9818\n",
      "after 17700 training steps, the loss is 0.0121612, the validation accuracy is 0.9836\n",
      "after 17800 training steps, the loss is 0.0165963, the validation accuracy is 0.9828\n",
      "after 17900 training steps, the loss is 0.0209153, the validation accuracy is 0.984\n",
      "after 18000 training steps, the loss is 0.0238452, the validation accuracy is 0.9814\n",
      "after 18100 training steps, the loss is 0.0159457, the validation accuracy is 0.9828\n",
      "after 18200 training steps, the loss is 0.0250696, the validation accuracy is 0.982\n",
      "after 18300 training steps, the loss is 0.012953, the validation accuracy is 0.9822\n",
      "after 18400 training steps, the loss is 0.0190619, the validation accuracy is 0.9824\n",
      "after 18500 training steps, the loss is 0.0167436, the validation accuracy is 0.9838\n",
      "after 18600 training steps, the loss is 0.0125924, the validation accuracy is 0.9834\n",
      "after 18700 training steps, the loss is 0.0227786, the validation accuracy is 0.9828\n",
      "after 18800 training steps, the loss is 0.0136067, the validation accuracy is 0.9842\n",
      "after 18900 training steps, the loss is 0.0162267, the validation accuracy is 0.9844\n",
      "after 19000 training steps, the loss is 0.0151378, the validation accuracy is 0.983\n",
      "after 19100 training steps, the loss is 0.0113503, the validation accuracy is 0.9824\n",
      "after 19200 training steps, the loss is 0.0141006, the validation accuracy is 0.983\n",
      "after 19300 training steps, the loss is 0.018333, the validation accuracy is 0.9834\n",
      "after 19400 training steps, the loss is 0.0157976, the validation accuracy is 0.9824\n",
      "after 19500 training steps, the loss is 0.0201733, the validation accuracy is 0.9822\n",
      "after 19600 training steps, the loss is 0.0159068, the validation accuracy is 0.9834\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19700 training steps, the loss is 0.019878, the validation accuracy is 0.9816\n",
      "after 19800 training steps, the loss is 0.0153434, the validation accuracy is 0.9832\n",
      "after 19900 training steps, the loss is 0.0146257, the validation accuracy is 0.9824\n",
      "after 20000 training steps, the loss is 0.0144622, the validation accuracy is 0.9822\n",
      "the training is finish!\n",
      "the train accuarcy is: 0.9998364\n",
      "the test accuarcy is: 0.9807\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=128, training_step=20000, lr=0.1, lambda_flag='l2', lambda_value=0.01, initial_way=MSRA, \n",
    "      hidden1=1000, hidden2=1000, hidden3=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "也没有什么起色，看来达到0.99以上是无望了，最后试试不增加隐层而直接在一层隐藏层上添加神经元数量～"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.215231, the validation accuracy is 0.9178\n",
      "after 200 training steps, the loss is 0.246613, the validation accuracy is 0.9314\n",
      "after 300 training steps, the loss is 0.315848, the validation accuracy is 0.9418\n",
      "after 400 training steps, the loss is 0.144525, the validation accuracy is 0.9514\n",
      "after 500 training steps, the loss is 0.0381219, the validation accuracy is 0.9554\n",
      "after 600 training steps, the loss is 0.521918, the validation accuracy is 0.9528\n",
      "after 700 training steps, the loss is 0.0373952, the validation accuracy is 0.9592\n",
      "after 800 training steps, the loss is 0.0648656, the validation accuracy is 0.9548\n",
      "after 900 training steps, the loss is 0.0928239, the validation accuracy is 0.9676\n",
      "after 1000 training steps, the loss is 0.187732, the validation accuracy is 0.9592\n",
      "after 1100 training steps, the loss is 0.219077, the validation accuracy is 0.9632\n",
      "after 1200 training steps, the loss is 0.0548509, the validation accuracy is 0.9638\n",
      "after 1300 training steps, the loss is 0.100286, the validation accuracy is 0.9688\n",
      "after 1400 training steps, the loss is 0.0298352, the validation accuracy is 0.9662\n",
      "after 1500 training steps, the loss is 0.0238302, the validation accuracy is 0.965\n",
      "after 1600 training steps, the loss is 0.100412, the validation accuracy is 0.968\n",
      "after 1700 training steps, the loss is 0.0215592, the validation accuracy is 0.9708\n",
      "after 1800 training steps, the loss is 0.0658808, the validation accuracy is 0.9736\n",
      "after 1900 training steps, the loss is 0.0526059, the validation accuracy is 0.9702\n",
      "after 2000 training steps, the loss is 0.165613, the validation accuracy is 0.972\n",
      "after 2100 training steps, the loss is 0.0305604, the validation accuracy is 0.9708\n",
      "after 2200 training steps, the loss is 0.172167, the validation accuracy is 0.9612\n",
      "after 2300 training steps, the loss is 0.443864, the validation accuracy is 0.9734\n",
      "after 2400 training steps, the loss is 0.183736, the validation accuracy is 0.9732\n",
      "after 2500 training steps, the loss is 0.00528224, the validation accuracy is 0.9752\n",
      "after 2600 training steps, the loss is 0.0834999, the validation accuracy is 0.9742\n",
      "after 2700 training steps, the loss is 0.0336068, the validation accuracy is 0.975\n",
      "after 2800 training steps, the loss is 0.142644, the validation accuracy is 0.9744\n",
      "after 2900 training steps, the loss is 0.024531, the validation accuracy is 0.9758\n",
      "after 3000 training steps, the loss is 0.0579754, the validation accuracy is 0.9758\n",
      "after 3100 training steps, the loss is 0.0306771, the validation accuracy is 0.9712\n",
      "after 3200 training steps, the loss is 0.0600185, the validation accuracy is 0.976\n",
      "after 3300 training steps, the loss is 0.0309521, the validation accuracy is 0.9794\n",
      "after 3400 training steps, the loss is 0.179571, the validation accuracy is 0.9764\n",
      "after 3500 training steps, the loss is 0.0184036, the validation accuracy is 0.9768\n",
      "after 3600 training steps, the loss is 0.164526, the validation accuracy is 0.9754\n",
      "after 3700 training steps, the loss is 0.0724865, the validation accuracy is 0.9742\n",
      "after 3800 training steps, the loss is 0.0578968, the validation accuracy is 0.9776\n",
      "after 3900 training steps, the loss is 0.0281103, the validation accuracy is 0.9756\n",
      "after 4000 training steps, the loss is 0.0598186, the validation accuracy is 0.979\n",
      "after 4100 training steps, the loss is 0.0311843, the validation accuracy is 0.9758\n",
      "after 4200 training steps, the loss is 0.00250096, the validation accuracy is 0.978\n",
      "after 4300 training steps, the loss is 0.0111595, the validation accuracy is 0.9792\n",
      "after 4400 training steps, the loss is 0.0177595, the validation accuracy is 0.9782\n",
      "after 4500 training steps, the loss is 0.00587872, the validation accuracy is 0.9792\n",
      "after 4600 training steps, the loss is 0.0325773, the validation accuracy is 0.977\n",
      "after 4700 training steps, the loss is 0.0149941, the validation accuracy is 0.9802\n",
      "after 4800 training steps, the loss is 0.00215668, the validation accuracy is 0.9786\n",
      "after 4900 training steps, the loss is 0.036931, the validation accuracy is 0.9796\n",
      "after 5000 training steps, the loss is 0.255998, the validation accuracy is 0.9812\n",
      "after 5100 training steps, the loss is 0.00479638, the validation accuracy is 0.98\n",
      "after 5200 training steps, the loss is 0.0124156, the validation accuracy is 0.9792\n",
      "after 5300 training steps, the loss is 0.0283389, the validation accuracy is 0.979\n",
      "after 5400 training steps, the loss is 0.0263545, the validation accuracy is 0.979\n",
      "after 5500 training steps, the loss is 0.0642977, the validation accuracy is 0.9788\n",
      "after 5600 training steps, the loss is 0.0793408, the validation accuracy is 0.9792\n",
      "after 5700 training steps, the loss is 0.00485112, the validation accuracy is 0.9812\n",
      "after 5800 training steps, the loss is 0.0108939, the validation accuracy is 0.9812\n",
      "after 5900 training steps, the loss is 0.00691979, the validation accuracy is 0.9826\n",
      "after 6000 training steps, the loss is 0.00509719, the validation accuracy is 0.9826\n",
      "after 6100 training steps, the loss is 0.00192648, the validation accuracy is 0.9806\n",
      "after 6200 training steps, the loss is 0.00782877, the validation accuracy is 0.9832\n",
      "after 6300 training steps, the loss is 0.00892314, the validation accuracy is 0.9808\n",
      "after 6400 training steps, the loss is 0.00675949, the validation accuracy is 0.9824\n",
      "after 6500 training steps, the loss is 0.0236397, the validation accuracy is 0.982\n",
      "after 6600 training steps, the loss is 0.00570744, the validation accuracy is 0.9814\n",
      "after 6700 training steps, the loss is 0.00945115, the validation accuracy is 0.9798\n",
      "after 6800 training steps, the loss is 0.0111649, the validation accuracy is 0.9812\n",
      "after 6900 training steps, the loss is 0.00546765, the validation accuracy is 0.9812\n",
      "after 7000 training steps, the loss is 0.0454823, the validation accuracy is 0.9812\n",
      "after 7100 training steps, the loss is 0.0249958, the validation accuracy is 0.9816\n",
      "after 7200 training steps, the loss is 0.0162954, the validation accuracy is 0.9822\n",
      "after 7300 training steps, the loss is 0.00805516, the validation accuracy is 0.9808\n",
      "after 7400 training steps, the loss is 0.00859787, the validation accuracy is 0.9802\n",
      "after 7500 training steps, the loss is 0.00366856, the validation accuracy is 0.9808\n",
      "after 7600 training steps, the loss is 0.0144385, the validation accuracy is 0.982\n",
      "after 7700 training steps, the loss is 0.00100226, the validation accuracy is 0.9826\n",
      "after 7800 training steps, the loss is 0.00780454, the validation accuracy is 0.9816\n",
      "after 7900 training steps, the loss is 0.00647611, the validation accuracy is 0.9818\n",
      "after 8000 training steps, the loss is 0.014035, the validation accuracy is 0.9828\n",
      "after 8100 training steps, the loss is 0.00913774, the validation accuracy is 0.981\n",
      "after 8200 training steps, the loss is 0.00289614, the validation accuracy is 0.983\n",
      "after 8300 training steps, the loss is 0.00633325, the validation accuracy is 0.9828\n",
      "after 8400 training steps, the loss is 0.00242163, the validation accuracy is 0.9816\n",
      "after 8500 training steps, the loss is 0.00275504, the validation accuracy is 0.9816\n",
      "after 8600 training steps, the loss is 0.00174288, the validation accuracy is 0.9818\n",
      "after 8700 training steps, the loss is 0.00514802, the validation accuracy is 0.9824\n",
      "after 8800 training steps, the loss is 0.00568943, the validation accuracy is 0.9826\n",
      "after 8900 training steps, the loss is 0.00121858, the validation accuracy is 0.9814\n",
      "after 9000 training steps, the loss is 0.044739, the validation accuracy is 0.9826\n",
      "after 9100 training steps, the loss is 0.0131126, the validation accuracy is 0.9834\n",
      "after 9200 training steps, the loss is 0.00388132, the validation accuracy is 0.983\n",
      "after 9300 training steps, the loss is 0.00926594, the validation accuracy is 0.9832\n",
      "after 9400 training steps, the loss is 0.0143218, the validation accuracy is 0.9836\n",
      "after 9500 training steps, the loss is 0.00661573, the validation accuracy is 0.9826\n",
      "after 9600 training steps, the loss is 0.00438468, the validation accuracy is 0.9824\n",
      "after 9700 training steps, the loss is 0.00147512, the validation accuracy is 0.9844\n",
      "after 9800 training steps, the loss is 0.000560154, the validation accuracy is 0.9838\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 9900 training steps, the loss is 0.000601007, the validation accuracy is 0.984\n",
      "after 10000 training steps, the loss is 0.00281077, the validation accuracy is 0.984\n",
      "after 10100 training steps, the loss is 0.00180228, the validation accuracy is 0.9844\n",
      "after 10200 training steps, the loss is 0.000293042, the validation accuracy is 0.9846\n",
      "after 10300 training steps, the loss is 0.00311566, the validation accuracy is 0.9828\n",
      "after 10400 training steps, the loss is 0.0179258, the validation accuracy is 0.9844\n",
      "after 10500 training steps, the loss is 0.0027312, the validation accuracy is 0.9832\n",
      "after 10600 training steps, the loss is 0.00369084, the validation accuracy is 0.984\n",
      "after 10700 training steps, the loss is 0.00182125, the validation accuracy is 0.9834\n",
      "after 10800 training steps, the loss is 0.00289831, the validation accuracy is 0.984\n",
      "after 10900 training steps, the loss is 0.00595131, the validation accuracy is 0.9832\n",
      "after 11000 training steps, the loss is 0.000195671, the validation accuracy is 0.9856\n",
      "after 11100 training steps, the loss is 0.00104589, the validation accuracy is 0.9834\n",
      "after 11200 training steps, the loss is 0.00174043, the validation accuracy is 0.9848\n",
      "after 11300 training steps, the loss is 0.00207031, the validation accuracy is 0.9844\n",
      "after 11400 training steps, the loss is 0.00182484, the validation accuracy is 0.985\n",
      "after 11500 training steps, the loss is 0.0133683, the validation accuracy is 0.9848\n",
      "after 11600 training steps, the loss is 0.00737476, the validation accuracy is 0.9834\n",
      "after 11700 training steps, the loss is 0.00189049, the validation accuracy is 0.9844\n",
      "after 11800 training steps, the loss is 0.00127232, the validation accuracy is 0.9844\n",
      "after 11900 training steps, the loss is 0.00240452, the validation accuracy is 0.9846\n",
      "after 12000 training steps, the loss is 0.0118415, the validation accuracy is 0.985\n",
      "after 12100 training steps, the loss is 0.00209606, the validation accuracy is 0.9838\n",
      "after 12200 training steps, the loss is 0.00108291, the validation accuracy is 0.985\n",
      "after 12300 training steps, the loss is 0.00182976, the validation accuracy is 0.984\n",
      "after 12400 training steps, the loss is 0.00560185, the validation accuracy is 0.9834\n",
      "after 12500 training steps, the loss is 0.00156638, the validation accuracy is 0.9856\n",
      "after 12600 training steps, the loss is 0.00211491, the validation accuracy is 0.9848\n",
      "after 12700 training steps, the loss is 0.000848486, the validation accuracy is 0.9836\n",
      "after 12800 training steps, the loss is 0.00535251, the validation accuracy is 0.9842\n",
      "after 12900 training steps, the loss is 0.00109948, the validation accuracy is 0.9844\n",
      "after 13000 training steps, the loss is 0.00278819, the validation accuracy is 0.9856\n",
      "after 13100 training steps, the loss is 0.00510307, the validation accuracy is 0.9848\n",
      "after 13200 training steps, the loss is 0.00138461, the validation accuracy is 0.985\n",
      "after 13300 training steps, the loss is 0.00534326, the validation accuracy is 0.985\n",
      "after 13400 training steps, the loss is 0.00212623, the validation accuracy is 0.9848\n",
      "after 13500 training steps, the loss is 0.000906539, the validation accuracy is 0.9854\n",
      "after 13600 training steps, the loss is 0.000153325, the validation accuracy is 0.985\n",
      "after 13700 training steps, the loss is 0.000897774, the validation accuracy is 0.9846\n",
      "after 13800 training steps, the loss is 0.00199369, the validation accuracy is 0.9852\n",
      "after 13900 training steps, the loss is 0.000561244, the validation accuracy is 0.9848\n",
      "after 14000 training steps, the loss is 0.00189412, the validation accuracy is 0.9848\n",
      "after 14100 training steps, the loss is 0.00123223, the validation accuracy is 0.9854\n",
      "after 14200 training steps, the loss is 0.000159547, the validation accuracy is 0.9848\n",
      "after 14300 training steps, the loss is 0.000366976, the validation accuracy is 0.9848\n",
      "after 14400 training steps, the loss is 0.00218798, the validation accuracy is 0.9852\n",
      "after 14500 training steps, the loss is 0.00113667, the validation accuracy is 0.9852\n",
      "after 14600 training steps, the loss is 0.00235641, the validation accuracy is 0.9856\n",
      "after 14700 training steps, the loss is 0.00296569, the validation accuracy is 0.9856\n",
      "after 14800 training steps, the loss is 0.000482202, the validation accuracy is 0.985\n",
      "after 14900 training steps, the loss is 0.00112168, the validation accuracy is 0.9848\n",
      "after 15000 training steps, the loss is 0.000593336, the validation accuracy is 0.9848\n",
      "after 15100 training steps, the loss is 0.00275167, the validation accuracy is 0.9846\n",
      "after 15200 training steps, the loss is 0.00112554, the validation accuracy is 0.9842\n",
      "after 15300 training steps, the loss is 0.00060834, the validation accuracy is 0.9854\n",
      "after 15400 training steps, the loss is 0.00013308, the validation accuracy is 0.985\n",
      "after 15500 training steps, the loss is 0.00288064, the validation accuracy is 0.985\n",
      "after 15600 training steps, the loss is 0.00106361, the validation accuracy is 0.9852\n",
      "after 15700 training steps, the loss is 0.000340778, the validation accuracy is 0.9854\n",
      "after 15800 training steps, the loss is 0.00124303, the validation accuracy is 0.9854\n",
      "after 15900 training steps, the loss is 0.0026883, the validation accuracy is 0.9844\n",
      "after 16000 training steps, the loss is 0.000399392, the validation accuracy is 0.9852\n",
      "after 16100 training steps, the loss is 0.000393689, the validation accuracy is 0.985\n",
      "after 16200 training steps, the loss is 0.000508214, the validation accuracy is 0.985\n",
      "after 16300 training steps, the loss is 6.21201e-05, the validation accuracy is 0.9856\n",
      "after 16400 training steps, the loss is 0.00238302, the validation accuracy is 0.985\n",
      "after 16500 training steps, the loss is 0.000746815, the validation accuracy is 0.9854\n",
      "after 16600 training steps, the loss is 0.00120786, the validation accuracy is 0.985\n",
      "after 16700 training steps, the loss is 0.00130392, the validation accuracy is 0.9854\n",
      "after 16800 training steps, the loss is 0.00065794, the validation accuracy is 0.9852\n",
      "after 16900 training steps, the loss is 7.55765e-05, the validation accuracy is 0.9852\n",
      "after 17000 training steps, the loss is 4.46885e-05, the validation accuracy is 0.9852\n",
      "after 17100 training steps, the loss is 0.00136309, the validation accuracy is 0.9858\n",
      "after 17200 training steps, the loss is 0.00151156, the validation accuracy is 0.985\n",
      "after 17300 training steps, the loss is 0.0126908, the validation accuracy is 0.9848\n",
      "after 17400 training steps, the loss is 0.00145209, the validation accuracy is 0.9842\n",
      "after 17500 training steps, the loss is 0.00179802, the validation accuracy is 0.9852\n",
      "after 17600 training steps, the loss is 0.00118373, the validation accuracy is 0.9854\n",
      "after 17700 training steps, the loss is 0.00181757, the validation accuracy is 0.9848\n",
      "after 17800 training steps, the loss is 0.00115296, the validation accuracy is 0.9854\n",
      "after 17900 training steps, the loss is 0.000697512, the validation accuracy is 0.9854\n",
      "after 18000 training steps, the loss is 6.35731e-05, the validation accuracy is 0.9846\n",
      "after 18100 training steps, the loss is 0.00157285, the validation accuracy is 0.9856\n",
      "after 18200 training steps, the loss is 0.000850249, the validation accuracy is 0.985\n",
      "after 18300 training steps, the loss is 0.000632601, the validation accuracy is 0.9854\n",
      "after 18400 training steps, the loss is 0.00281323, the validation accuracy is 0.9852\n",
      "after 18500 training steps, the loss is 0.000726011, the validation accuracy is 0.9848\n",
      "after 18600 training steps, the loss is 0.00145351, the validation accuracy is 0.9848\n",
      "after 18700 training steps, the loss is 0.000264413, the validation accuracy is 0.9854\n",
      "after 18800 training steps, the loss is 0.00178579, the validation accuracy is 0.9852\n",
      "after 18900 training steps, the loss is 0.000620757, the validation accuracy is 0.9852\n",
      "after 19000 training steps, the loss is 0.000514093, the validation accuracy is 0.985\n",
      "after 19100 training steps, the loss is 0.00134992, the validation accuracy is 0.985\n",
      "after 19200 training steps, the loss is 0.00209721, the validation accuracy is 0.9852\n",
      "after 19300 training steps, the loss is 0.000463984, the validation accuracy is 0.9854\n",
      "after 19400 training steps, the loss is 0.00105971, the validation accuracy is 0.9852\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19500 training steps, the loss is 0.00130326, the validation accuracy is 0.9856\n",
      "after 19600 training steps, the loss is 0.00166144, the validation accuracy is 0.985\n",
      "after 19700 training steps, the loss is 0.000467594, the validation accuracy is 0.9856\n",
      "after 19800 training steps, the loss is 0.000410316, the validation accuracy is 0.9856\n",
      "after 19900 training steps, the loss is 0.000343194, the validation accuracy is 0.9852\n",
      "after 20000 training steps, the loss is 0.000868001, the validation accuracy is 0.9852\n",
      "the training is finish!\n",
      "the train accuarcy is: 1.0\n",
      "the test accuarcy is: 0.9841\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=20000, lr=0.3, hidden1=2000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这样效果竟然是最好的。。。继续加大数量试试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 100 training steps, the loss is 0.711609, the validation accuracy is 0.909\n",
      "after 200 training steps, the loss is 0.0144847, the validation accuracy is 0.9446\n",
      "after 300 training steps, the loss is 0.426769, the validation accuracy is 0.9468\n",
      "after 400 training steps, the loss is 0.0947704, the validation accuracy is 0.9466\n",
      "after 500 training steps, the loss is 0.0294169, the validation accuracy is 0.952\n",
      "after 600 training steps, the loss is 0.0209194, the validation accuracy is 0.9584\n",
      "after 700 training steps, the loss is 0.0831345, the validation accuracy is 0.963\n",
      "after 800 training steps, the loss is 0.087108, the validation accuracy is 0.9644\n",
      "after 900 training steps, the loss is 0.0965173, the validation accuracy is 0.9624\n",
      "after 1000 training steps, the loss is 0.143213, the validation accuracy is 0.9686\n",
      "after 1100 training steps, the loss is 0.0955209, the validation accuracy is 0.964\n",
      "after 1200 training steps, the loss is 0.0930269, the validation accuracy is 0.9682\n",
      "after 1300 training steps, the loss is 0.0393525, the validation accuracy is 0.9682\n",
      "after 1400 training steps, the loss is 0.261302, the validation accuracy is 0.9726\n",
      "after 1500 training steps, the loss is 0.00979928, the validation accuracy is 0.9648\n",
      "after 1600 training steps, the loss is 0.109101, the validation accuracy is 0.9682\n",
      "after 1700 training steps, the loss is 0.00447953, the validation accuracy is 0.973\n",
      "after 1800 training steps, the loss is 0.00291605, the validation accuracy is 0.9774\n",
      "after 1900 training steps, the loss is 0.136691, the validation accuracy is 0.9716\n",
      "after 2000 training steps, the loss is 0.0475202, the validation accuracy is 0.9716\n",
      "after 2100 training steps, the loss is 0.19682, the validation accuracy is 0.9714\n",
      "after 2200 training steps, the loss is 0.0566884, the validation accuracy is 0.9732\n",
      "after 2300 training steps, the loss is 0.0751582, the validation accuracy is 0.9758\n",
      "after 2400 training steps, the loss is 0.0687422, the validation accuracy is 0.9758\n",
      "after 2500 training steps, the loss is 0.0263516, the validation accuracy is 0.9788\n",
      "after 2600 training steps, the loss is 0.0380138, the validation accuracy is 0.981\n",
      "after 2700 training steps, the loss is 0.0102072, the validation accuracy is 0.9796\n",
      "after 2800 training steps, the loss is 0.00159331, the validation accuracy is 0.978\n",
      "after 2900 training steps, the loss is 0.00546875, the validation accuracy is 0.981\n",
      "after 3000 training steps, the loss is 0.0104087, the validation accuracy is 0.978\n",
      "after 3100 training steps, the loss is 0.00149838, the validation accuracy is 0.9816\n",
      "after 3200 training steps, the loss is 0.00277793, the validation accuracy is 0.9788\n",
      "after 3300 training steps, the loss is 0.00860586, the validation accuracy is 0.9792\n",
      "after 3400 training steps, the loss is 0.00385901, the validation accuracy is 0.9804\n",
      "after 3500 training steps, the loss is 0.163665, the validation accuracy is 0.9806\n",
      "after 3600 training steps, the loss is 0.00798679, the validation accuracy is 0.9794\n",
      "after 3700 training steps, the loss is 0.0356716, the validation accuracy is 0.9818\n",
      "after 3800 training steps, the loss is 0.0456333, the validation accuracy is 0.9812\n",
      "after 3900 training steps, the loss is 0.00152139, the validation accuracy is 0.9818\n",
      "after 4000 training steps, the loss is 0.00371302, the validation accuracy is 0.981\n",
      "after 4100 training steps, the loss is 0.00718803, the validation accuracy is 0.98\n",
      "after 4200 training steps, the loss is 0.0146332, the validation accuracy is 0.9808\n",
      "after 4300 training steps, the loss is 0.00134592, the validation accuracy is 0.9832\n",
      "after 4400 training steps, the loss is 0.0205733, the validation accuracy is 0.9838\n",
      "after 4500 training steps, the loss is 0.000393033, the validation accuracy is 0.9836\n",
      "after 4600 training steps, the loss is 0.0134986, the validation accuracy is 0.984\n",
      "after 4700 training steps, the loss is 0.0023936, the validation accuracy is 0.9828\n",
      "after 4800 training steps, the loss is 0.00140102, the validation accuracy is 0.9844\n",
      "after 4900 training steps, the loss is 0.0280524, the validation accuracy is 0.9828\n",
      "after 5000 training steps, the loss is 0.00888597, the validation accuracy is 0.9826\n",
      "after 5100 training steps, the loss is 0.000476811, the validation accuracy is 0.9816\n",
      "after 5200 training steps, the loss is 0.0119144, the validation accuracy is 0.9844\n",
      "after 5300 training steps, the loss is 0.00228966, the validation accuracy is 0.984\n",
      "after 5400 training steps, the loss is 0.00453264, the validation accuracy is 0.9814\n",
      "after 5500 training steps, the loss is 0.00121862, the validation accuracy is 0.9816\n",
      "after 5600 training steps, the loss is 0.000584131, the validation accuracy is 0.9838\n",
      "after 5700 training steps, the loss is 0.000940741, the validation accuracy is 0.9834\n",
      "after 5800 training steps, the loss is 0.000865681, the validation accuracy is 0.9834\n",
      "after 5900 training steps, the loss is 0.00179773, the validation accuracy is 0.9844\n",
      "after 6000 training steps, the loss is 0.00128622, the validation accuracy is 0.9838\n",
      "after 6100 training steps, the loss is 0.000826525, the validation accuracy is 0.9846\n",
      "after 6200 training steps, the loss is 0.0250182, the validation accuracy is 0.9834\n",
      "after 6300 training steps, the loss is 0.000815426, the validation accuracy is 0.9836\n",
      "after 6400 training steps, the loss is 0.000934966, the validation accuracy is 0.984\n",
      "after 6500 training steps, the loss is 0.00279455, the validation accuracy is 0.984\n",
      "after 6600 training steps, the loss is 0.00346737, the validation accuracy is 0.9822\n",
      "after 6700 training steps, the loss is 0.00233803, the validation accuracy is 0.9844\n",
      "after 6800 training steps, the loss is 0.000796626, the validation accuracy is 0.984\n",
      "after 6900 training steps, the loss is 0.00340712, the validation accuracy is 0.9842\n",
      "after 7000 training steps, the loss is 0.000487375, the validation accuracy is 0.9848\n",
      "after 7100 training steps, the loss is 0.00200307, the validation accuracy is 0.9848\n",
      "after 7200 training steps, the loss is 0.000147128, the validation accuracy is 0.9856\n",
      "after 7300 training steps, the loss is 0.00406313, the validation accuracy is 0.9848\n",
      "after 7400 training steps, the loss is 0.00707787, the validation accuracy is 0.9846\n",
      "after 7500 training steps, the loss is 0.00139857, the validation accuracy is 0.9846\n",
      "after 7600 training steps, the loss is 0.00058197, the validation accuracy is 0.985\n",
      "after 7700 training steps, the loss is 0.00133338, the validation accuracy is 0.9848\n",
      "after 7800 training steps, the loss is 0.000714526, the validation accuracy is 0.9848\n",
      "after 7900 training steps, the loss is 0.00505981, the validation accuracy is 0.985\n",
      "after 8000 training steps, the loss is 0.000785668, the validation accuracy is 0.985\n",
      "after 8100 training steps, the loss is 0.000535474, the validation accuracy is 0.9848\n",
      "after 8200 training steps, the loss is 0.000620922, the validation accuracy is 0.9846\n",
      "after 8300 training steps, the loss is 0.000493217, the validation accuracy is 0.9844\n",
      "after 8400 training steps, the loss is 0.00499489, the validation accuracy is 0.9844\n",
      "after 8500 training steps, the loss is 0.000634665, the validation accuracy is 0.9846\n",
      "after 8600 training steps, the loss is 0.000964329, the validation accuracy is 0.9846\n",
      "after 8700 training steps, the loss is 0.0008225, the validation accuracy is 0.9848\n",
      "after 8800 training steps, the loss is 0.000462376, the validation accuracy is 0.985\n",
      "after 8900 training steps, the loss is 0.000848797, the validation accuracy is 0.9848\n",
      "after 9000 training steps, the loss is 0.00108162, the validation accuracy is 0.9848\n",
      "after 9100 training steps, the loss is 0.00140427, the validation accuracy is 0.985\n",
      "after 9200 training steps, the loss is 0.00675718, the validation accuracy is 0.9854\n",
      "after 9300 training steps, the loss is 0.00262917, the validation accuracy is 0.9846\n",
      "after 9400 training steps, the loss is 0.000660999, the validation accuracy is 0.9848\n",
      "after 9500 training steps, the loss is 0.000805774, the validation accuracy is 0.9852\n",
      "after 9600 training steps, the loss is 0.000806465, the validation accuracy is 0.9856\n",
      "after 9700 training steps, the loss is 0.00685389, the validation accuracy is 0.9858\n",
      "after 9800 training steps, the loss is 0.000940508, the validation accuracy is 0.9858\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 9900 training steps, the loss is 0.000545736, the validation accuracy is 0.9852\n",
      "after 10000 training steps, the loss is 0.000206721, the validation accuracy is 0.9846\n",
      "after 10100 training steps, the loss is 0.000169242, the validation accuracy is 0.985\n",
      "after 10200 training steps, the loss is 0.00262242, the validation accuracy is 0.9846\n",
      "after 10300 training steps, the loss is 0.00137458, the validation accuracy is 0.9848\n",
      "after 10400 training steps, the loss is 0.00297256, the validation accuracy is 0.985\n",
      "after 10500 training steps, the loss is 0.0015808, the validation accuracy is 0.985\n",
      "after 10600 training steps, the loss is 0.00149466, the validation accuracy is 0.9844\n",
      "after 10700 training steps, the loss is 0.000826336, the validation accuracy is 0.9846\n",
      "after 10800 training steps, the loss is 0.000155924, the validation accuracy is 0.9854\n",
      "after 10900 training steps, the loss is 0.00204231, the validation accuracy is 0.985\n",
      "after 11000 training steps, the loss is 0.00103348, the validation accuracy is 0.9844\n",
      "after 11100 training steps, the loss is 9.32143e-05, the validation accuracy is 0.9844\n",
      "after 11200 training steps, the loss is 0.000867752, the validation accuracy is 0.9842\n",
      "after 11300 training steps, the loss is 0.000424848, the validation accuracy is 0.9846\n",
      "after 11400 training steps, the loss is 0.00046378, the validation accuracy is 0.985\n",
      "after 11500 training steps, the loss is 0.00130954, the validation accuracy is 0.985\n",
      "after 11600 training steps, the loss is 0.000536847, the validation accuracy is 0.9848\n",
      "after 11700 training steps, the loss is 0.000179611, the validation accuracy is 0.985\n",
      "after 11800 training steps, the loss is 0.00130062, the validation accuracy is 0.985\n",
      "after 11900 training steps, the loss is 0.00159832, the validation accuracy is 0.985\n",
      "after 12000 training steps, the loss is 0.000185817, the validation accuracy is 0.9852\n",
      "after 12100 training steps, the loss is 0.000849622, the validation accuracy is 0.9844\n",
      "after 12200 training steps, the loss is 0.000156812, the validation accuracy is 0.9846\n",
      "after 12300 training steps, the loss is 0.000599102, the validation accuracy is 0.985\n",
      "after 12400 training steps, the loss is 0.000576185, the validation accuracy is 0.9846\n",
      "after 12500 training steps, the loss is 0.000475452, the validation accuracy is 0.985\n",
      "after 12600 training steps, the loss is 0.00115309, the validation accuracy is 0.985\n",
      "after 12700 training steps, the loss is 0.00115307, the validation accuracy is 0.985\n",
      "after 12800 training steps, the loss is 0.000938994, the validation accuracy is 0.9848\n",
      "after 12900 training steps, the loss is 0.000240005, the validation accuracy is 0.9848\n",
      "after 13000 training steps, the loss is 0.0004837, the validation accuracy is 0.9846\n",
      "after 13100 training steps, the loss is 0.00166129, the validation accuracy is 0.9848\n",
      "after 13200 training steps, the loss is 0.00188411, the validation accuracy is 0.985\n",
      "after 13300 training steps, the loss is 0.000659119, the validation accuracy is 0.985\n",
      "after 13400 training steps, the loss is 0.000248057, the validation accuracy is 0.9852\n",
      "after 13500 training steps, the loss is 0.000931093, the validation accuracy is 0.9852\n",
      "after 13600 training steps, the loss is 0.00047304, the validation accuracy is 0.985\n",
      "after 13700 training steps, the loss is 0.000997274, the validation accuracy is 0.985\n",
      "after 13800 training steps, the loss is 0.000375317, the validation accuracy is 0.9846\n",
      "after 13900 training steps, the loss is 0.000920306, the validation accuracy is 0.9852\n",
      "after 14000 training steps, the loss is 0.000922614, the validation accuracy is 0.9844\n",
      "after 14100 training steps, the loss is 0.0013251, the validation accuracy is 0.9846\n",
      "after 14200 training steps, the loss is 8.22919e-05, the validation accuracy is 0.985\n",
      "after 14300 training steps, the loss is 0.00185718, the validation accuracy is 0.9848\n",
      "after 14400 training steps, the loss is 0.000148637, the validation accuracy is 0.9848\n",
      "after 14500 training steps, the loss is 0.000818503, the validation accuracy is 0.985\n",
      "after 14600 training steps, the loss is 0.000788843, the validation accuracy is 0.9848\n",
      "after 14700 training steps, the loss is 0.000278718, the validation accuracy is 0.9852\n",
      "after 14800 training steps, the loss is 0.00119796, the validation accuracy is 0.9852\n",
      "after 14900 training steps, the loss is 0.000121237, the validation accuracy is 0.9846\n",
      "after 15000 training steps, the loss is 0.00161634, the validation accuracy is 0.9848\n",
      "after 15100 training steps, the loss is 0.000150891, the validation accuracy is 0.9846\n",
      "after 15200 training steps, the loss is 0.000464519, the validation accuracy is 0.9846\n",
      "after 15300 training steps, the loss is 0.00175184, the validation accuracy is 0.9846\n",
      "after 15400 training steps, the loss is 0.000388577, the validation accuracy is 0.9846\n",
      "after 15500 training steps, the loss is 6.80561e-05, the validation accuracy is 0.9848\n",
      "after 15600 training steps, the loss is 0.000130851, the validation accuracy is 0.9848\n",
      "after 15700 training steps, the loss is 0.00068603, the validation accuracy is 0.9846\n",
      "after 15800 training steps, the loss is 0.00117576, the validation accuracy is 0.9846\n",
      "after 15900 training steps, the loss is 0.00155928, the validation accuracy is 0.9856\n",
      "after 16000 training steps, the loss is 0.000140899, the validation accuracy is 0.9848\n",
      "after 16100 training steps, the loss is 0.000184659, the validation accuracy is 0.9858\n",
      "after 16200 training steps, the loss is 0.00083819, the validation accuracy is 0.9858\n",
      "after 16300 training steps, the loss is 0.00169748, the validation accuracy is 0.9854\n",
      "after 16400 training steps, the loss is 0.00160636, the validation accuracy is 0.9852\n",
      "after 16500 training steps, the loss is 0.000233737, the validation accuracy is 0.9856\n",
      "after 16600 training steps, the loss is 0.000340281, the validation accuracy is 0.985\n",
      "after 16700 training steps, the loss is 0.000132522, the validation accuracy is 0.9854\n",
      "after 16800 training steps, the loss is 0.000866323, the validation accuracy is 0.9846\n",
      "after 16900 training steps, the loss is 0.000559302, the validation accuracy is 0.9848\n",
      "after 17000 training steps, the loss is 0.00113298, the validation accuracy is 0.9848\n",
      "after 17100 training steps, the loss is 0.000780705, the validation accuracy is 0.985\n",
      "after 17200 training steps, the loss is 0.000847533, the validation accuracy is 0.9848\n",
      "after 17300 training steps, the loss is 0.000481533, the validation accuracy is 0.985\n",
      "after 17400 training steps, the loss is 0.00083659, the validation accuracy is 0.9848\n",
      "after 17500 training steps, the loss is 0.000427268, the validation accuracy is 0.985\n",
      "after 17600 training steps, the loss is 0.000424437, the validation accuracy is 0.9848\n",
      "after 17700 training steps, the loss is 0.000969941, the validation accuracy is 0.9848\n",
      "after 17800 training steps, the loss is 0.000346835, the validation accuracy is 0.9858\n",
      "after 17900 training steps, the loss is 0.000219195, the validation accuracy is 0.9846\n",
      "after 18000 training steps, the loss is 0.000638674, the validation accuracy is 0.9848\n",
      "after 18100 training steps, the loss is 0.000121149, the validation accuracy is 0.9848\n",
      "after 18200 training steps, the loss is 0.000101539, the validation accuracy is 0.9848\n",
      "after 18300 training steps, the loss is 0.000167857, the validation accuracy is 0.985\n",
      "after 18400 training steps, the loss is 0.000545423, the validation accuracy is 0.9852\n",
      "after 18500 training steps, the loss is 0.00112631, the validation accuracy is 0.9852\n",
      "after 18600 training steps, the loss is 0.000234262, the validation accuracy is 0.985\n",
      "after 18700 training steps, the loss is 0.000848157, the validation accuracy is 0.9848\n",
      "after 18800 training steps, the loss is 0.000919444, the validation accuracy is 0.9848\n",
      "after 18900 training steps, the loss is 0.000464064, the validation accuracy is 0.9846\n",
      "after 19000 training steps, the loss is 0.000404462, the validation accuracy is 0.9844\n",
      "after 19100 training steps, the loss is 0.000127103, the validation accuracy is 0.9846\n",
      "after 19200 training steps, the loss is 0.000915941, the validation accuracy is 0.9846\n",
      "after 19300 training steps, the loss is 0.000810001, the validation accuracy is 0.9844\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 19400 training steps, the loss is 0.000630382, the validation accuracy is 0.985\n",
      "after 19500 training steps, the loss is 0.00249099, the validation accuracy is 0.9848\n",
      "after 19600 training steps, the loss is 0.000102912, the validation accuracy is 0.985\n",
      "after 19700 training steps, the loss is 0.00101332, the validation accuracy is 0.9846\n",
      "after 19800 training steps, the loss is 0.000385215, the validation accuracy is 0.9852\n",
      "after 19900 training steps, the loss is 0.000517768, the validation accuracy is 0.985\n",
      "after 20000 training steps, the loss is 0.000775266, the validation accuracy is 0.9848\n",
      "the training is finish!\n",
      "the train accuarcy is: 1.0\n",
      "the test accuarcy is: 0.9851\n"
     ]
    }
   ],
   "source": [
    "train(batch_size=32, training_step=20000, lr=0.3, hidden1=10000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "加大神经元数目后，模型效果又提升了0.001，得到了目前最好模型。在输出的验证集准确率中发现准确率后面一直在0.985左右震荡，猜测是学习率不够小，稍后考虑将学习率改为随着迭代次数增加自适应减小。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 总结"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 调整迭代次数、学习率，增加神经网络隐层数目和神经元个数是比较有效的调优方式，调节这几个参数都能使模型最终在验证集上的准确率稳定在0.98以上。其中最有效的方式是调节迭代次数和神经元数量，一个隐层其实就够了。\n",
    "* 正则化参数有效缩小了训练集和测试集的准确率之间的距离，但对模型最终效果没有提升。\n",
    "* 随着神经网络层数增加，根据激活函数选择适合的参数初始化方式比较有效。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
